1 /******************************************************************************
2 * balloon.c
3 *
4 * Xen balloon driver - enables returning/claiming memory to/from Xen.
5 *
6 * Copyright (c) 2003, B Dragovic
7 * Copyright (c) 2003-2004, M Williamson, K Fraser
8 * Copyright (c) 2005 Dan M. Smith, IBM Corporation
9 *
10 * This file may be distributed separately from the Linux kernel, or
11 * incorporated into other software packages, subject to the following license:
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a copy
14 * of this source file (the "Software"), to deal in the Software without
15 * restriction, including without limitation the rights to use, copy, modify,
16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
17 * and to permit persons to whom the Software is furnished to do so, subject to
18 * the following conditions:
19 *
20 * The above copyright notice and this permission notice shall be included in
21 * all copies or substantial portions of the Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 * IN THE SOFTWARE.
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD: stable/9/sys/dev/xen/balloon/balloon.c 249132 2013-04-05 08:22:11Z mav $");
34
35 #include <sys/param.h>
36 #include <sys/lock.h>
37 #include <sys/kernel.h>
38 #include <sys/kthread.h>
39 #include <sys/malloc.h>
40 #include <sys/mutex.h>
41 #include <sys/sysctl.h>
42
43 #include <machine/xen/xen-os.h>
44 #include <machine/xen/xenvar.h>
45 #include <machine/xen/xenfunc.h>
46 #include <xen/hypervisor.h>
47 #include <xen/xenstore/xenstorevar.h>
48
49 #include <vm/vm.h>
50 #include <vm/vm_page.h>
51
52 static MALLOC_DEFINE(M_BALLOON, "Balloon", "Xen Balloon Driver");
53
54 struct mtx balloon_mutex;
55
56 /*
57 * Protects atomic reservation decrease/increase against concurrent increases.
58 * Also protects non-atomic updates of current_pages and driver_pages, and
59 * balloon lists.
60 */
61 struct mtx balloon_lock;
62
63 /* We increase/decrease in batches which fit in a page */
64 static unsigned long frame_list[PAGE_SIZE / sizeof(unsigned long)];
65 #define ARRAY_SIZE(A) (sizeof(A) / sizeof(A[0]))
66
67 struct balloon_stats {
68 /* We aim for 'current allocation' == 'target allocation'. */
69 unsigned long current_pages;
70 unsigned long target_pages;
71 /* We may hit the hard limit in Xen. If we do then we remember it. */
72 unsigned long hard_limit;
73 /*
74 * Drivers may alter the memory reservation independently, but they
75 * must inform the balloon driver so we avoid hitting the hard limit.
76 */
77 unsigned long driver_pages;
78 /* Number of pages in high- and low-memory balloons. */
79 unsigned long balloon_low;
80 unsigned long balloon_high;
81 };
82
83 static struct balloon_stats balloon_stats;
84 #define bs balloon_stats
85
86 SYSCTL_DECL(_dev_xen);
87 static SYSCTL_NODE(_dev_xen, OID_AUTO, balloon, CTLFLAG_RD, NULL, "Balloon");
88 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, current, CTLFLAG_RD,
89 &bs.current_pages, 0, "Current allocation");
90 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, target, CTLFLAG_RD,
91 &bs.target_pages, 0, "Target allocation");
92 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, driver_pages, CTLFLAG_RD,
93 &bs.driver_pages, 0, "Driver pages");
94 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, hard_limit, CTLFLAG_RD,
95 &bs.hard_limit, 0, "Xen hard limit");
96 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, low_mem, CTLFLAG_RD,
97 &bs.balloon_low, 0, "Low-mem balloon");
98 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, high_mem, CTLFLAG_RD,
99 &bs.balloon_high, 0, "High-mem balloon");
100
101 struct balloon_entry {
102 vm_page_t page;
103 STAILQ_ENTRY(balloon_entry) list;
104 };
105
106 /* List of ballooned pages, threaded through the mem_map array. */
107 static STAILQ_HEAD(,balloon_entry) ballooned_pages;
108
109 /* Main work function, always executed in process context. */
110 static void balloon_process(void *unused);
111
112 #define IPRINTK(fmt, args...) \
113 printk(KERN_INFO "xen_mem: " fmt, ##args)
114 #define WPRINTK(fmt, args...) \
115 printk(KERN_WARNING "xen_mem: " fmt, ##args)
116
117 /* balloon_append: add the given page to the balloon. */
118 static void
balloon_append(vm_page_t page)119 balloon_append(vm_page_t page)
120 {
121 struct balloon_entry *entry;
122
123 entry = malloc(sizeof(struct balloon_entry), M_BALLOON, M_WAITOK);
124 entry->page = page;
125 STAILQ_INSERT_HEAD(&ballooned_pages, entry, list);
126 bs.balloon_low++;
127 }
128
129 /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
130 static vm_page_t
balloon_retrieve(void)131 balloon_retrieve(void)
132 {
133 vm_page_t page;
134 struct balloon_entry *entry;
135
136 if (STAILQ_EMPTY(&ballooned_pages))
137 return NULL;
138
139 entry = STAILQ_FIRST(&ballooned_pages);
140 STAILQ_REMOVE_HEAD(&ballooned_pages, list);
141
142 page = entry->page;
143 free(entry, M_BALLOON);
144
145 bs.balloon_low--;
146
147 return page;
148 }
149
150 static unsigned long
current_target(void)151 current_target(void)
152 {
153 unsigned long target = min(bs.target_pages, bs.hard_limit);
154 if (target > (bs.current_pages + bs.balloon_low + bs.balloon_high))
155 target = bs.current_pages + bs.balloon_low + bs.balloon_high;
156 return target;
157 }
158
159 static unsigned long
minimum_target(void)160 minimum_target(void)
161 {
162 #ifdef XENHVM
163 #define max_pfn physmem
164 #else
165 #define max_pfn HYPERVISOR_shared_info->arch.max_pfn
166 #endif
167 unsigned long min_pages, curr_pages = current_target();
168
169 #define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
170 /* Simple continuous piecewiese linear function:
171 * max MiB -> min MiB gradient
172 * 0 0
173 * 16 16
174 * 32 24
175 * 128 72 (1/2)
176 * 512 168 (1/4)
177 * 2048 360 (1/8)
178 * 8192 552 (1/32)
179 * 32768 1320
180 * 131072 4392
181 */
182 if (max_pfn < MB2PAGES(128))
183 min_pages = MB2PAGES(8) + (max_pfn >> 1);
184 else if (max_pfn < MB2PAGES(512))
185 min_pages = MB2PAGES(40) + (max_pfn >> 2);
186 else if (max_pfn < MB2PAGES(2048))
187 min_pages = MB2PAGES(104) + (max_pfn >> 3);
188 else
189 min_pages = MB2PAGES(296) + (max_pfn >> 5);
190 #undef MB2PAGES
191
192 /* Don't enforce growth */
193 return min(min_pages, curr_pages);
194 #ifndef CONFIG_XEN
195 #undef max_pfn
196 #endif
197 }
198
199 static int
increase_reservation(unsigned long nr_pages)200 increase_reservation(unsigned long nr_pages)
201 {
202 unsigned long pfn, i;
203 struct balloon_entry *entry;
204 vm_page_t page;
205 long rc;
206 struct xen_memory_reservation reservation = {
207 .address_bits = 0,
208 .extent_order = 0,
209 .domid = DOMID_SELF
210 };
211
212 if (nr_pages > ARRAY_SIZE(frame_list))
213 nr_pages = ARRAY_SIZE(frame_list);
214
215 mtx_lock(&balloon_lock);
216
217 for (entry = STAILQ_FIRST(&ballooned_pages), i = 0;
218 i < nr_pages; i++, entry = STAILQ_NEXT(entry, list)) {
219 KASSERT(entry, ("ballooned_pages list corrupt"));
220 page = entry->page;
221 frame_list[i] = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT);
222 }
223
224 set_xen_guest_handle(reservation.extent_start, frame_list);
225 reservation.nr_extents = nr_pages;
226 rc = HYPERVISOR_memory_op(
227 XENMEM_populate_physmap, &reservation);
228 if (rc < nr_pages) {
229 if (rc > 0) {
230 int ret;
231
232 /* We hit the Xen hard limit: reprobe. */
233 reservation.nr_extents = rc;
234 ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
235 &reservation);
236 KASSERT(ret == rc, ("HYPERVISOR_memory_op failed"));
237 }
238 if (rc >= 0)
239 bs.hard_limit = (bs.current_pages + rc -
240 bs.driver_pages);
241 goto out;
242 }
243
244 for (i = 0; i < nr_pages; i++) {
245 page = balloon_retrieve();
246 KASSERT(page, ("balloon_retrieve failed"));
247
248 pfn = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT);
249 KASSERT((xen_feature(XENFEAT_auto_translated_physmap) ||
250 !phys_to_machine_mapping_valid(pfn)),
251 ("auto translated physmap but mapping is valid"));
252
253 set_phys_to_machine(pfn, frame_list[i]);
254
255 #if 0
256 #ifndef XENHVM
257 /* Link back into the page tables if not highmem. */
258 if (pfn < max_low_pfn) {
259 int ret;
260 ret = HYPERVISOR_update_va_mapping(
261 (unsigned long)__va(pfn << PAGE_SHIFT),
262 pfn_pte_ma(frame_list[i], PAGE_KERNEL),
263 0);
264 PASSING(ret == 0,
265 ("HYPERVISOR_update_va_mapping failed"));
266 }
267 #endif
268 #endif
269
270 /* Relinquish the page back to the allocator. */
271 vm_page_unwire(page, 0);
272 vm_page_free(page);
273 }
274
275 bs.current_pages += nr_pages;
276 //totalram_pages = bs.current_pages;
277
278 out:
279 mtx_unlock(&balloon_lock);
280
281 return 0;
282 }
283
284 static int
decrease_reservation(unsigned long nr_pages)285 decrease_reservation(unsigned long nr_pages)
286 {
287 unsigned long pfn, i;
288 vm_page_t page;
289 int need_sleep = 0;
290 int ret;
291 struct xen_memory_reservation reservation = {
292 .address_bits = 0,
293 .extent_order = 0,
294 .domid = DOMID_SELF
295 };
296
297 if (nr_pages > ARRAY_SIZE(frame_list))
298 nr_pages = ARRAY_SIZE(frame_list);
299
300 for (i = 0; i < nr_pages; i++) {
301 if ((page = vm_page_alloc(NULL, 0,
302 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
303 VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
304 nr_pages = i;
305 need_sleep = 1;
306 break;
307 }
308
309 pfn = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT);
310 frame_list[i] = PFNTOMFN(pfn);
311
312 #if 0
313 if (!PageHighMem(page)) {
314 v = phys_to_virt(pfn << PAGE_SHIFT);
315 scrub_pages(v, 1);
316 #ifdef CONFIG_XEN
317 ret = HYPERVISOR_update_va_mapping(
318 (unsigned long)v, __pte_ma(0), 0);
319 BUG_ON(ret);
320 #endif
321 }
322 #endif
323 #ifdef CONFIG_XEN_SCRUB_PAGES
324 else {
325 v = kmap(page);
326 scrub_pages(v, 1);
327 kunmap(page);
328 }
329 #endif
330 }
331
332 #ifdef CONFIG_XEN
333 /* Ensure that ballooned highmem pages don't have kmaps. */
334 kmap_flush_unused();
335 flush_tlb_all();
336 #endif
337
338 mtx_lock(&balloon_lock);
339
340 /* No more mappings: invalidate P2M and add to balloon. */
341 for (i = 0; i < nr_pages; i++) {
342 pfn = MFNTOPFN(frame_list[i]);
343 set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
344 balloon_append(PHYS_TO_VM_PAGE(pfn << PAGE_SHIFT));
345 }
346
347 set_xen_guest_handle(reservation.extent_start, frame_list);
348 reservation.nr_extents = nr_pages;
349 ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
350 KASSERT(ret == nr_pages, ("HYPERVISOR_memory_op failed"));
351
352 bs.current_pages -= nr_pages;
353 //totalram_pages = bs.current_pages;
354
355 mtx_unlock(&balloon_lock);
356
357 return (need_sleep);
358 }
359
360 /*
361 * We avoid multiple worker processes conflicting via the balloon mutex.
362 * We may of course race updates of the target counts (which are protected
363 * by the balloon lock), or with changes to the Xen hard limit, but we will
364 * recover from these in time.
365 */
366 static void
balloon_process(void * unused)367 balloon_process(void *unused)
368 {
369 int need_sleep = 0;
370 long credit;
371
372 mtx_lock(&balloon_mutex);
373 for (;;) {
374 int sleep_time;
375
376 do {
377 credit = current_target() - bs.current_pages;
378 if (credit > 0)
379 need_sleep = (increase_reservation(credit) != 0);
380 if (credit < 0)
381 need_sleep = (decrease_reservation(-credit) != 0);
382
383 } while ((credit != 0) && !need_sleep);
384
385 /* Schedule more work if there is some still to be done. */
386 if (current_target() != bs.current_pages)
387 sleep_time = hz;
388 else
389 sleep_time = 0;
390
391 msleep(balloon_process, &balloon_mutex, 0, "balloon",
392 sleep_time);
393 }
394 mtx_unlock(&balloon_mutex);
395 }
396
397 /* Resets the Xen limit, sets new target, and kicks off processing. */
398 static void
set_new_target(unsigned long target)399 set_new_target(unsigned long target)
400 {
401 /* No need for lock. Not read-modify-write updates. */
402 bs.hard_limit = ~0UL;
403 bs.target_pages = max(target, minimum_target());
404 wakeup(balloon_process);
405 }
406
407 static struct xs_watch target_watch =
408 {
409 .node = "memory/target"
410 };
411
412 /* React to a change in the target key */
413 static void
watch_target(struct xs_watch * watch,const char ** vec,unsigned int len)414 watch_target(struct xs_watch *watch,
415 const char **vec, unsigned int len)
416 {
417 unsigned long long new_target;
418 int err;
419
420 err = xs_scanf(XST_NIL, "memory", "target", NULL,
421 "%llu", &new_target);
422 if (err) {
423 /* This is ok (for domain0 at least) - so just return */
424 return;
425 }
426
427 /* The given memory/target value is in KiB, so it needs converting to
428 pages. PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
429 */
430 set_new_target(new_target >> (PAGE_SHIFT - 10));
431
432 }
433
434 static void
balloon_init_watcher(void * arg)435 balloon_init_watcher(void *arg)
436 {
437 int err;
438
439 if (!is_running_on_xen())
440 return;
441
442 err = xs_register_watch(&target_watch);
443 if (err)
444 printf("Failed to set balloon watcher\n");
445
446 }
447 SYSINIT(balloon_init_watcher, SI_SUB_PSEUDO, SI_ORDER_ANY,
448 balloon_init_watcher, NULL);
449
450 static void
balloon_init(void * arg)451 balloon_init(void *arg)
452 {
453 #ifndef XENHVM
454 vm_page_t page;
455 unsigned long pfn;
456
457 #define max_pfn HYPERVISOR_shared_info->arch.max_pfn
458 #endif
459
460 if (!is_running_on_xen())
461 return;
462
463 mtx_init(&balloon_lock, "balloon_lock", NULL, MTX_DEF);
464 mtx_init(&balloon_mutex, "balloon_mutex", NULL, MTX_DEF);
465
466 #ifndef XENHVM
467 bs.current_pages = min(xen_start_info->nr_pages, max_pfn);
468 #else
469 bs.current_pages = physmem;
470 #endif
471 bs.target_pages = bs.current_pages;
472 bs.balloon_low = 0;
473 bs.balloon_high = 0;
474 bs.driver_pages = 0UL;
475 bs.hard_limit = ~0UL;
476
477 kproc_create(balloon_process, NULL, NULL, 0, 0, "balloon");
478
479 #ifndef XENHVM
480 /* Initialise the balloon with excess memory space. */
481 for (pfn = xen_start_info->nr_pages; pfn < max_pfn; pfn++) {
482 page = PHYS_TO_VM_PAGE(pfn << PAGE_SHIFT);
483 balloon_append(page);
484 }
485 #undef max_pfn
486 #endif
487
488 target_watch.callback = watch_target;
489
490 return;
491 }
492 SYSINIT(balloon_init, SI_SUB_PSEUDO, SI_ORDER_ANY, balloon_init, NULL);
493
494 void balloon_update_driver_allowance(long delta);
495
496 void
balloon_update_driver_allowance(long delta)497 balloon_update_driver_allowance(long delta)
498 {
499 mtx_lock(&balloon_lock);
500 bs.driver_pages += delta;
501 mtx_unlock(&balloon_lock);
502 }
503
504 #if 0
505 static int dealloc_pte_fn(
506 pte_t *pte, struct page *pte_page, unsigned long addr, void *data)
507 {
508 unsigned long mfn = pte_mfn(*pte);
509 int ret;
510 struct xen_memory_reservation reservation = {
511 .extent_start = &mfn,
512 .nr_extents = 1,
513 .extent_order = 0,
514 .domid = DOMID_SELF
515 };
516 set_pte_at(&init_mm, addr, pte, __pte_ma(0));
517 set_phys_to_machine(__pa(addr) >> PAGE_SHIFT, INVALID_P2M_ENTRY);
518 ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
519 KASSERT(ret == 1, ("HYPERVISOR_memory_op failed"));
520 return 0;
521 }
522
523 #endif
524
525 #if 0
526 vm_page_t
527 balloon_alloc_empty_page_range(unsigned long nr_pages)
528 {
529 vm_page_t pages;
530 int i, rc;
531 unsigned long *mfn_list;
532 struct xen_memory_reservation reservation = {
533 .address_bits = 0,
534 .extent_order = 0,
535 .domid = DOMID_SELF
536 };
537
538 pages = vm_page_alloc_contig(nr_pages, 0, -1, 4, 4)
539 if (pages == NULL)
540 return NULL;
541
542 mfn_list = malloc(nr_pages*sizeof(unsigned long), M_DEVBUF, M_WAITOK);
543
544 for (i = 0; i < nr_pages; i++) {
545 mfn_list[i] = PFNTOMFN(VM_PAGE_TO_PHYS(pages[i]) >> PAGE_SHIFT);
546 PFNTOMFN(i) = INVALID_P2M_ENTRY;
547 reservation.extent_start = mfn_list;
548 reservation.nr_extents = nr_pages;
549 rc = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
550 &reservation);
551 KASSERT(rc == nr_pages, ("HYPERVISOR_memory_op failed"));
552 }
553
554 current_pages -= nr_pages;
555
556 wakeup(balloon_process);
557
558 return pages;
559 }
560
561 void
562 balloon_dealloc_empty_page_range(vm_page_t page, unsigned long nr_pages)
563 {
564 unsigned long i;
565
566 for (i = 0; i < nr_pages; i++)
567 balloon_append(page + i);
568
569 wakeup(balloon_process);
570 }
571 #endif
572