1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /*
26 * Copyright (c) 2013, 2019 by Delphix. All rights reserved.
27 * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved.
28 */
29
30 #include <sys/zfs_context.h>
31 #include <sys/spa.h>
32 #include <sys/dmu.h>
33 #include <sys/dnode.h>
34 #include <sys/zio.h>
35 #include <sys/range_tree.h>
36
37 /*
38 * Range trees are tree-based data structures that can be used to
39 * track free space or generally any space allocation information.
40 * A range tree keeps track of individual segments and automatically
41 * provides facilities such as adjacent extent merging and extent
42 * splitting in response to range add/remove requests.
43 *
44 * A range tree starts out completely empty, with no segments in it.
45 * Adding an allocation via range_tree_add to the range tree can either:
46 * 1) create a new extent
47 * 2) extend an adjacent extent
48 * 3) merge two adjacent extents
49 * Conversely, removing an allocation via range_tree_remove can:
50 * 1) completely remove an extent
51 * 2) shorten an extent (if the allocation was near one of its ends)
52 * 3) split an extent into two extents, in effect punching a hole
53 *
54 * A range tree is also capable of 'bridging' gaps when adding
55 * allocations. This is useful for cases when close proximity of
56 * allocations is an important detail that needs to be represented
57 * in the range tree. See range_tree_set_gap(). The default behavior
58 * is not to bridge gaps (i.e. the maximum allowed gap size is 0).
59 *
60 * In order to traverse a range tree, use either the range_tree_walk()
61 * or range_tree_vacate() functions.
62 *
63 * To obtain more accurate information on individual segment
64 * operations that the range tree performs "under the hood", you can
65 * specify a set of callbacks by passing a range_tree_ops_t structure
66 * to the range_tree_create function. Any callbacks that are non-NULL
67 * are then called at the appropriate times.
68 *
69 * The range tree code also supports a special variant of range trees
70 * that can bridge small gaps between segments. This kind of tree is used
71 * by the dsl scanning code to group I/Os into mostly sequential chunks to
72 * optimize disk performance. The code here attempts to do this with as
73 * little memory and computational overhead as possible. One limitation of
74 * this implementation is that segments of range trees with gaps can only
75 * support removing complete segments.
76 */
77
78 static inline void
rs_copy(range_seg_t * src,range_seg_t * dest,range_tree_t * rt)79 rs_copy(range_seg_t *src, range_seg_t *dest, range_tree_t *rt)
80 {
81 ASSERT3U(rt->rt_type, <=, RANGE_SEG_NUM_TYPES);
82 size_t size = 0;
83 switch (rt->rt_type) {
84 case RANGE_SEG32:
85 size = sizeof (range_seg32_t);
86 break;
87 case RANGE_SEG64:
88 size = sizeof (range_seg64_t);
89 break;
90 case RANGE_SEG_GAP:
91 size = sizeof (range_seg_gap_t);
92 break;
93 default:
94 VERIFY(0);
95 }
96 bcopy(src, dest, size);
97 }
98
99 void
range_tree_stat_verify(range_tree_t * rt)100 range_tree_stat_verify(range_tree_t *rt)
101 {
102 range_seg_t *rs;
103 zfs_btree_index_t where;
104 uint64_t hist[RANGE_TREE_HISTOGRAM_SIZE] = { 0 };
105 int i;
106
107 for (rs = zfs_btree_first(&rt->rt_root, &where); rs != NULL;
108 rs = zfs_btree_next(&rt->rt_root, &where, &where)) {
109 uint64_t size = rs_get_end(rs, rt) - rs_get_start(rs, rt);
110 int idx = highbit64(size) - 1;
111
112 hist[idx]++;
113 ASSERT3U(hist[idx], !=, 0);
114 }
115
116 for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
117 if (hist[i] != rt->rt_histogram[i]) {
118 zfs_dbgmsg("i=%d, hist=%px, hist=%llu, rt_hist=%llu",
119 i, hist, (u_longlong_t)hist[i],
120 (u_longlong_t)rt->rt_histogram[i]);
121 }
122 VERIFY3U(hist[i], ==, rt->rt_histogram[i]);
123 }
124 }
125
126 static void
range_tree_stat_incr(range_tree_t * rt,range_seg_t * rs)127 range_tree_stat_incr(range_tree_t *rt, range_seg_t *rs)
128 {
129 uint64_t size = rs_get_end(rs, rt) - rs_get_start(rs, rt);
130 int idx = highbit64(size) - 1;
131
132 ASSERT(size != 0);
133 ASSERT3U(idx, <,
134 sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));
135
136 rt->rt_histogram[idx]++;
137 ASSERT3U(rt->rt_histogram[idx], !=, 0);
138 }
139
140 static void
range_tree_stat_decr(range_tree_t * rt,range_seg_t * rs)141 range_tree_stat_decr(range_tree_t *rt, range_seg_t *rs)
142 {
143 uint64_t size = rs_get_end(rs, rt) - rs_get_start(rs, rt);
144 int idx = highbit64(size) - 1;
145
146 ASSERT(size != 0);
147 ASSERT3U(idx, <,
148 sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));
149
150 ASSERT3U(rt->rt_histogram[idx], !=, 0);
151 rt->rt_histogram[idx]--;
152 }
153
154 static int
range_tree_seg32_compare(const void * x1,const void * x2)155 range_tree_seg32_compare(const void *x1, const void *x2)
156 {
157 const range_seg32_t *r1 = x1;
158 const range_seg32_t *r2 = x2;
159
160 ASSERT3U(r1->rs_start, <=, r1->rs_end);
161 ASSERT3U(r2->rs_start, <=, r2->rs_end);
162
163 return ((r1->rs_start >= r2->rs_end) - (r1->rs_end <= r2->rs_start));
164 }
165
166 static int
range_tree_seg64_compare(const void * x1,const void * x2)167 range_tree_seg64_compare(const void *x1, const void *x2)
168 {
169 const range_seg64_t *r1 = x1;
170 const range_seg64_t *r2 = x2;
171
172 ASSERT3U(r1->rs_start, <=, r1->rs_end);
173 ASSERT3U(r2->rs_start, <=, r2->rs_end);
174
175 return ((r1->rs_start >= r2->rs_end) - (r1->rs_end <= r2->rs_start));
176 }
177
178 static int
range_tree_seg_gap_compare(const void * x1,const void * x2)179 range_tree_seg_gap_compare(const void *x1, const void *x2)
180 {
181 const range_seg_gap_t *r1 = x1;
182 const range_seg_gap_t *r2 = x2;
183
184 ASSERT3U(r1->rs_start, <=, r1->rs_end);
185 ASSERT3U(r2->rs_start, <=, r2->rs_end);
186
187 return ((r1->rs_start >= r2->rs_end) - (r1->rs_end <= r2->rs_start));
188 }
189
190 range_tree_t *
range_tree_create_gap(const range_tree_ops_t * ops,range_seg_type_t type,void * arg,uint64_t start,uint64_t shift,uint64_t gap)191 range_tree_create_gap(const range_tree_ops_t *ops, range_seg_type_t type,
192 void *arg, uint64_t start, uint64_t shift, uint64_t gap)
193 {
194 range_tree_t *rt = kmem_zalloc(sizeof (range_tree_t), KM_SLEEP);
195
196 ASSERT3U(shift, <, 64);
197 ASSERT3U(type, <=, RANGE_SEG_NUM_TYPES);
198 size_t size;
199 int (*compare) (const void *, const void *);
200 switch (type) {
201 case RANGE_SEG32:
202 size = sizeof (range_seg32_t);
203 compare = range_tree_seg32_compare;
204 break;
205 case RANGE_SEG64:
206 size = sizeof (range_seg64_t);
207 compare = range_tree_seg64_compare;
208 break;
209 case RANGE_SEG_GAP:
210 size = sizeof (range_seg_gap_t);
211 compare = range_tree_seg_gap_compare;
212 break;
213 default:
214 panic("Invalid range seg type %d", type);
215 }
216 zfs_btree_create(&rt->rt_root, compare, size);
217
218 rt->rt_ops = ops;
219 rt->rt_gap = gap;
220 rt->rt_arg = arg;
221 rt->rt_type = type;
222 rt->rt_start = start;
223 rt->rt_shift = shift;
224
225 if (rt->rt_ops != NULL && rt->rt_ops->rtop_create != NULL)
226 rt->rt_ops->rtop_create(rt, rt->rt_arg);
227
228 return (rt);
229 }
230
231 range_tree_t *
range_tree_create(const range_tree_ops_t * ops,range_seg_type_t type,void * arg,uint64_t start,uint64_t shift)232 range_tree_create(const range_tree_ops_t *ops, range_seg_type_t type,
233 void *arg, uint64_t start, uint64_t shift)
234 {
235 return (range_tree_create_gap(ops, type, arg, start, shift, 0));
236 }
237
238 void
range_tree_destroy(range_tree_t * rt)239 range_tree_destroy(range_tree_t *rt)
240 {
241 VERIFY0(rt->rt_space);
242
243 if (rt->rt_ops != NULL && rt->rt_ops->rtop_destroy != NULL)
244 rt->rt_ops->rtop_destroy(rt, rt->rt_arg);
245
246 zfs_btree_destroy(&rt->rt_root);
247 kmem_free(rt, sizeof (*rt));
248 }
249
250 void
range_tree_adjust_fill(range_tree_t * rt,range_seg_t * rs,int64_t delta)251 range_tree_adjust_fill(range_tree_t *rt, range_seg_t *rs, int64_t delta)
252 {
253 if (delta < 0 && delta * -1 >= rs_get_fill(rs, rt)) {
254 zfs_panic_recover("zfs: attempting to decrease fill to or "
255 "below 0; probable double remove in segment [%llx:%llx]",
256 (longlong_t)rs_get_start(rs, rt),
257 (longlong_t)rs_get_end(rs, rt));
258 }
259 if (rs_get_fill(rs, rt) + delta > rs_get_end(rs, rt) -
260 rs_get_start(rs, rt)) {
261 zfs_panic_recover("zfs: attempting to increase fill beyond "
262 "max; probable double add in segment [%llx:%llx]",
263 (longlong_t)rs_get_start(rs, rt),
264 (longlong_t)rs_get_end(rs, rt));
265 }
266
267 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
268 rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
269 rs_set_fill(rs, rt, rs_get_fill(rs, rt) + delta);
270 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
271 rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
272 }
273
274 static void
range_tree_add_impl(void * arg,uint64_t start,uint64_t size,uint64_t fill)275 range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill)
276 {
277 range_tree_t *rt = arg;
278 zfs_btree_index_t where;
279 range_seg_t *rs_before, *rs_after, *rs;
280 range_seg_max_t tmp, rsearch;
281 uint64_t end = start + size, gap = rt->rt_gap;
282 uint64_t bridge_size = 0;
283 boolean_t merge_before, merge_after;
284
285 ASSERT3U(size, !=, 0);
286 ASSERT3U(fill, <=, size);
287 ASSERT3U(start + size, >, start);
288
289 rs_set_start(&rsearch, rt, start);
290 rs_set_end(&rsearch, rt, end);
291 rs = zfs_btree_find(&rt->rt_root, &rsearch, &where);
292
293 /*
294 * If this is a gap-supporting range tree, it is possible that we
295 * are inserting into an existing segment. In this case simply
296 * bump the fill count and call the remove / add callbacks. If the
297 * new range will extend an existing segment, we remove the
298 * existing one, apply the new extent to it and re-insert it using
299 * the normal code paths.
300 */
301 if (rs != NULL) {
302 if (gap == 0) {
303 zfs_panic_recover("zfs: adding existent segment to "
304 "range tree (offset=%llx size=%llx)",
305 (longlong_t)start, (longlong_t)size);
306 return;
307 }
308 uint64_t rstart = rs_get_start(rs, rt);
309 uint64_t rend = rs_get_end(rs, rt);
310 if (rstart <= start && rend >= end) {
311 range_tree_adjust_fill(rt, rs, fill);
312 return;
313 }
314
315 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
316 rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
317
318 range_tree_stat_decr(rt, rs);
319 rt->rt_space -= rend - rstart;
320
321 fill += rs_get_fill(rs, rt);
322 start = MIN(start, rstart);
323 end = MAX(end, rend);
324 size = end - start;
325
326 zfs_btree_remove(&rt->rt_root, rs);
327 range_tree_add_impl(rt, start, size, fill);
328 return;
329 }
330
331 ASSERT3P(rs, ==, NULL);
332
333 /*
334 * Determine whether or not we will have to merge with our neighbors.
335 * If gap != 0, we might need to merge with our neighbors even if we
336 * aren't directly touching.
337 */
338 zfs_btree_index_t where_before, where_after;
339 rs_before = zfs_btree_prev(&rt->rt_root, &where, &where_before);
340 rs_after = zfs_btree_next(&rt->rt_root, &where, &where_after);
341
342 merge_before = (rs_before != NULL && rs_get_end(rs_before, rt) >=
343 start - gap);
344 merge_after = (rs_after != NULL && rs_get_start(rs_after, rt) <= end +
345 gap);
346
347 if (merge_before && gap != 0)
348 bridge_size += start - rs_get_end(rs_before, rt);
349 if (merge_after && gap != 0)
350 bridge_size += rs_get_start(rs_after, rt) - end;
351
352 if (merge_before && merge_after) {
353 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) {
354 rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg);
355 rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg);
356 }
357
358 range_tree_stat_decr(rt, rs_before);
359 range_tree_stat_decr(rt, rs_after);
360
361 rs_copy(rs_after, &tmp, rt);
362 uint64_t before_start = rs_get_start_raw(rs_before, rt);
363 uint64_t before_fill = rs_get_fill(rs_before, rt);
364 uint64_t after_fill = rs_get_fill(rs_after, rt);
365 zfs_btree_remove_idx(&rt->rt_root, &where_before);
366
367 /*
368 * We have to re-find the node because our old reference is
369 * invalid as soon as we do any mutating btree operations.
370 */
371 rs_after = zfs_btree_find(&rt->rt_root, &tmp, &where_after);
372 rs_set_start_raw(rs_after, rt, before_start);
373 rs_set_fill(rs_after, rt, after_fill + before_fill + fill);
374 rs = rs_after;
375 } else if (merge_before) {
376 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
377 rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg);
378
379 range_tree_stat_decr(rt, rs_before);
380
381 uint64_t before_fill = rs_get_fill(rs_before, rt);
382 rs_set_end(rs_before, rt, end);
383 rs_set_fill(rs_before, rt, before_fill + fill);
384 rs = rs_before;
385 } else if (merge_after) {
386 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
387 rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg);
388
389 range_tree_stat_decr(rt, rs_after);
390
391 uint64_t after_fill = rs_get_fill(rs_after, rt);
392 rs_set_start(rs_after, rt, start);
393 rs_set_fill(rs_after, rt, after_fill + fill);
394 rs = rs_after;
395 } else {
396 rs = &tmp;
397
398 rs_set_start(rs, rt, start);
399 rs_set_end(rs, rt, end);
400 rs_set_fill(rs, rt, fill);
401 zfs_btree_add_idx(&rt->rt_root, rs, &where);
402 }
403
404 if (gap != 0) {
405 ASSERT3U(rs_get_fill(rs, rt), <=, rs_get_end(rs, rt) -
406 rs_get_start(rs, rt));
407 } else {
408 ASSERT3U(rs_get_fill(rs, rt), ==, rs_get_end(rs, rt) -
409 rs_get_start(rs, rt));
410 }
411
412 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
413 rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
414
415 range_tree_stat_incr(rt, rs);
416 rt->rt_space += size + bridge_size;
417 }
418
419 void
range_tree_add(void * arg,uint64_t start,uint64_t size)420 range_tree_add(void *arg, uint64_t start, uint64_t size)
421 {
422 range_tree_add_impl(arg, start, size, size);
423 }
424
425 static void
range_tree_remove_impl(range_tree_t * rt,uint64_t start,uint64_t size,boolean_t do_fill)426 range_tree_remove_impl(range_tree_t *rt, uint64_t start, uint64_t size,
427 boolean_t do_fill)
428 {
429 zfs_btree_index_t where;
430 range_seg_t *rs;
431 range_seg_max_t rsearch, rs_tmp;
432 uint64_t end = start + size;
433 boolean_t left_over, right_over;
434
435 VERIFY3U(size, !=, 0);
436 VERIFY3U(size, <=, rt->rt_space);
437 if (rt->rt_type == RANGE_SEG64)
438 ASSERT3U(start + size, >, start);
439
440 rs_set_start(&rsearch, rt, start);
441 rs_set_end(&rsearch, rt, end);
442 rs = zfs_btree_find(&rt->rt_root, &rsearch, &where);
443
444 /* Make sure we completely overlap with someone */
445 if (rs == NULL) {
446 zfs_panic_recover("zfs: removing nonexistent segment from "
447 "range tree (offset=%llx size=%llx)",
448 (longlong_t)start, (longlong_t)size);
449 return;
450 }
451
452 /*
453 * Range trees with gap support must only remove complete segments
454 * from the tree. This allows us to maintain accurate fill accounting
455 * and to ensure that bridged sections are not leaked. If we need to
456 * remove less than the full segment, we can only adjust the fill count.
457 */
458 if (rt->rt_gap != 0) {
459 if (do_fill) {
460 if (rs_get_fill(rs, rt) == size) {
461 start = rs_get_start(rs, rt);
462 end = rs_get_end(rs, rt);
463 size = end - start;
464 } else {
465 range_tree_adjust_fill(rt, rs, -size);
466 return;
467 }
468 } else if (rs_get_start(rs, rt) != start ||
469 rs_get_end(rs, rt) != end) {
470 zfs_panic_recover("zfs: freeing partial segment of "
471 "gap tree (offset=%llx size=%llx) of "
472 "(offset=%llx size=%llx)",
473 (longlong_t)start, (longlong_t)size,
474 (longlong_t)rs_get_start(rs, rt),
475 (longlong_t)rs_get_end(rs, rt) - rs_get_start(rs,
476 rt));
477 return;
478 }
479 }
480
481 VERIFY3U(rs_get_start(rs, rt), <=, start);
482 VERIFY3U(rs_get_end(rs, rt), >=, end);
483
484 left_over = (rs_get_start(rs, rt) != start);
485 right_over = (rs_get_end(rs, rt) != end);
486
487 range_tree_stat_decr(rt, rs);
488
489 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
490 rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
491
492 if (left_over && right_over) {
493 range_seg_max_t newseg;
494 rs_set_start(&newseg, rt, end);
495 rs_set_end_raw(&newseg, rt, rs_get_end_raw(rs, rt));
496 rs_set_fill(&newseg, rt, rs_get_end(rs, rt) - end);
497 range_tree_stat_incr(rt, &newseg);
498
499 // This modifies the buffer already inside the range tree
500 rs_set_end(rs, rt, start);
501
502 rs_copy(rs, &rs_tmp, rt);
503 if (zfs_btree_next(&rt->rt_root, &where, &where) != NULL)
504 zfs_btree_add_idx(&rt->rt_root, &newseg, &where);
505 else
506 zfs_btree_add(&rt->rt_root, &newseg);
507
508 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
509 rt->rt_ops->rtop_add(rt, &newseg, rt->rt_arg);
510 } else if (left_over) {
511 // This modifies the buffer already inside the range tree
512 rs_set_end(rs, rt, start);
513 rs_copy(rs, &rs_tmp, rt);
514 } else if (right_over) {
515 // This modifies the buffer already inside the range tree
516 rs_set_start(rs, rt, end);
517 rs_copy(rs, &rs_tmp, rt);
518 } else {
519 zfs_btree_remove_idx(&rt->rt_root, &where);
520 rs = NULL;
521 }
522
523 if (rs != NULL) {
524 /*
525 * The fill of the leftover segment will always be equal to
526 * the size, since we do not support removing partial segments
527 * of range trees with gaps.
528 */
529 rs_set_fill_raw(rs, rt, rs_get_end_raw(rs, rt) -
530 rs_get_start_raw(rs, rt));
531 range_tree_stat_incr(rt, &rs_tmp);
532
533 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
534 rt->rt_ops->rtop_add(rt, &rs_tmp, rt->rt_arg);
535 }
536
537 rt->rt_space -= size;
538 }
539
540 void
range_tree_remove(void * arg,uint64_t start,uint64_t size)541 range_tree_remove(void *arg, uint64_t start, uint64_t size)
542 {
543 range_tree_remove_impl(arg, start, size, B_FALSE);
544 }
545
546 void
range_tree_remove_fill(range_tree_t * rt,uint64_t start,uint64_t size)547 range_tree_remove_fill(range_tree_t *rt, uint64_t start, uint64_t size)
548 {
549 range_tree_remove_impl(rt, start, size, B_TRUE);
550 }
551
552 void
range_tree_resize_segment(range_tree_t * rt,range_seg_t * rs,uint64_t newstart,uint64_t newsize)553 range_tree_resize_segment(range_tree_t *rt, range_seg_t *rs,
554 uint64_t newstart, uint64_t newsize)
555 {
556 int64_t delta = newsize - (rs_get_end(rs, rt) - rs_get_start(rs, rt));
557
558 range_tree_stat_decr(rt, rs);
559 if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
560 rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
561
562 rs_set_start(rs, rt, newstart);
563 rs_set_end(rs, rt, newstart + newsize);
564
565 range_tree_stat_incr(rt, rs);
566 if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
567 rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
568
569 rt->rt_space += delta;
570 }
571
572 static range_seg_t *
range_tree_find_impl(range_tree_t * rt,uint64_t start,uint64_t size)573 range_tree_find_impl(range_tree_t *rt, uint64_t start, uint64_t size)
574 {
575 range_seg_max_t rsearch;
576 uint64_t end = start + size;
577
578 VERIFY(size != 0);
579
580 rs_set_start(&rsearch, rt, start);
581 rs_set_end(&rsearch, rt, end);
582 return (zfs_btree_find(&rt->rt_root, &rsearch, NULL));
583 }
584
585 range_seg_t *
range_tree_find(range_tree_t * rt,uint64_t start,uint64_t size)586 range_tree_find(range_tree_t *rt, uint64_t start, uint64_t size)
587 {
588 if (rt->rt_type == RANGE_SEG64)
589 ASSERT3U(start + size, >, start);
590
591 range_seg_t *rs = range_tree_find_impl(rt, start, size);
592 if (rs != NULL && rs_get_start(rs, rt) <= start &&
593 rs_get_end(rs, rt) >= start + size) {
594 return (rs);
595 }
596 return (NULL);
597 }
598
599 void
range_tree_verify_not_present(range_tree_t * rt,uint64_t off,uint64_t size)600 range_tree_verify_not_present(range_tree_t *rt, uint64_t off, uint64_t size)
601 {
602 range_seg_t *rs = range_tree_find(rt, off, size);
603 if (rs != NULL)
604 panic("segment already in tree; rs=%p", (void *)rs);
605 }
606
607 boolean_t
range_tree_contains(range_tree_t * rt,uint64_t start,uint64_t size)608 range_tree_contains(range_tree_t *rt, uint64_t start, uint64_t size)
609 {
610 return (range_tree_find(rt, start, size) != NULL);
611 }
612
613 /*
614 * Returns the first subset of the given range which overlaps with the range
615 * tree. Returns true if there is a segment in the range, and false if there
616 * isn't.
617 */
618 boolean_t
range_tree_find_in(range_tree_t * rt,uint64_t start,uint64_t size,uint64_t * ostart,uint64_t * osize)619 range_tree_find_in(range_tree_t *rt, uint64_t start, uint64_t size,
620 uint64_t *ostart, uint64_t *osize)
621 {
622 if (rt->rt_type == RANGE_SEG64)
623 ASSERT3U(start + size, >, start);
624
625 range_seg_max_t rsearch;
626 rs_set_start(&rsearch, rt, start);
627 rs_set_end_raw(&rsearch, rt, rs_get_start_raw(&rsearch, rt) + 1);
628
629 zfs_btree_index_t where;
630 range_seg_t *rs = zfs_btree_find(&rt->rt_root, &rsearch, &where);
631 if (rs != NULL) {
632 *ostart = start;
633 *osize = MIN(size, rs_get_end(rs, rt) - start);
634 return (B_TRUE);
635 }
636
637 rs = zfs_btree_next(&rt->rt_root, &where, &where);
638 if (rs == NULL || rs_get_start(rs, rt) > start + size)
639 return (B_FALSE);
640
641 *ostart = rs_get_start(rs, rt);
642 *osize = MIN(start + size, rs_get_end(rs, rt)) -
643 rs_get_start(rs, rt);
644 return (B_TRUE);
645 }
646
647 /*
648 * Ensure that this range is not in the tree, regardless of whether
649 * it is currently in the tree.
650 */
651 void
range_tree_clear(range_tree_t * rt,uint64_t start,uint64_t size)652 range_tree_clear(range_tree_t *rt, uint64_t start, uint64_t size)
653 {
654 range_seg_t *rs;
655
656 if (size == 0)
657 return;
658
659 if (rt->rt_type == RANGE_SEG64)
660 ASSERT3U(start + size, >, start);
661
662 while ((rs = range_tree_find_impl(rt, start, size)) != NULL) {
663 uint64_t free_start = MAX(rs_get_start(rs, rt), start);
664 uint64_t free_end = MIN(rs_get_end(rs, rt), start + size);
665 range_tree_remove(rt, free_start, free_end - free_start);
666 }
667 }
668
669 void
range_tree_swap(range_tree_t ** rtsrc,range_tree_t ** rtdst)670 range_tree_swap(range_tree_t **rtsrc, range_tree_t **rtdst)
671 {
672 range_tree_t *rt;
673
674 ASSERT0(range_tree_space(*rtdst));
675 ASSERT0(zfs_btree_numnodes(&(*rtdst)->rt_root));
676
677 rt = *rtsrc;
678 *rtsrc = *rtdst;
679 *rtdst = rt;
680 }
681
682 void
range_tree_vacate(range_tree_t * rt,range_tree_func_t * func,void * arg)683 range_tree_vacate(range_tree_t *rt, range_tree_func_t *func, void *arg)
684 {
685 if (rt->rt_ops != NULL && rt->rt_ops->rtop_vacate != NULL)
686 rt->rt_ops->rtop_vacate(rt, rt->rt_arg);
687
688 if (func != NULL) {
689 range_seg_t *rs;
690 zfs_btree_index_t *cookie = NULL;
691
692 while ((rs = zfs_btree_destroy_nodes(&rt->rt_root, &cookie)) !=
693 NULL) {
694 func(arg, rs_get_start(rs, rt), rs_get_end(rs, rt) -
695 rs_get_start(rs, rt));
696 }
697 } else {
698 zfs_btree_clear(&rt->rt_root);
699 }
700
701 bzero(rt->rt_histogram, sizeof (rt->rt_histogram));
702 rt->rt_space = 0;
703 }
704
705 void
range_tree_walk(range_tree_t * rt,range_tree_func_t * func,void * arg)706 range_tree_walk(range_tree_t *rt, range_tree_func_t *func, void *arg)
707 {
708 zfs_btree_index_t where;
709 for (range_seg_t *rs = zfs_btree_first(&rt->rt_root, &where);
710 rs != NULL; rs = zfs_btree_next(&rt->rt_root, &where, &where)) {
711 func(arg, rs_get_start(rs, rt), rs_get_end(rs, rt) -
712 rs_get_start(rs, rt));
713 }
714 }
715
716 range_seg_t *
range_tree_first(range_tree_t * rt)717 range_tree_first(range_tree_t *rt)
718 {
719 return (zfs_btree_first(&rt->rt_root, NULL));
720 }
721
722 uint64_t
range_tree_space(range_tree_t * rt)723 range_tree_space(range_tree_t *rt)
724 {
725 return (rt->rt_space);
726 }
727
728 uint64_t
range_tree_numsegs(range_tree_t * rt)729 range_tree_numsegs(range_tree_t *rt)
730 {
731 return ((rt == NULL) ? 0 : zfs_btree_numnodes(&rt->rt_root));
732 }
733
734 boolean_t
range_tree_is_empty(range_tree_t * rt)735 range_tree_is_empty(range_tree_t *rt)
736 {
737 ASSERT(rt != NULL);
738 return (range_tree_space(rt) == 0);
739 }
740
741 /*
742 * Remove any overlapping ranges between the given segment [start, end)
743 * from removefrom. Add non-overlapping leftovers to addto.
744 */
745 void
range_tree_remove_xor_add_segment(uint64_t start,uint64_t end,range_tree_t * removefrom,range_tree_t * addto)746 range_tree_remove_xor_add_segment(uint64_t start, uint64_t end,
747 range_tree_t *removefrom, range_tree_t *addto)
748 {
749 zfs_btree_index_t where;
750 range_seg_max_t starting_rs;
751 rs_set_start(&starting_rs, removefrom, start);
752 rs_set_end_raw(&starting_rs, removefrom, rs_get_start_raw(&starting_rs,
753 removefrom) + 1);
754
755 range_seg_t *curr = zfs_btree_find(&removefrom->rt_root,
756 &starting_rs, &where);
757
758 if (curr == NULL)
759 curr = zfs_btree_next(&removefrom->rt_root, &where, &where);
760
761 range_seg_t *next;
762 for (; curr != NULL; curr = next) {
763 if (start == end)
764 return;
765 VERIFY3U(start, <, end);
766
767 /* there is no overlap */
768 if (end <= rs_get_start(curr, removefrom)) {
769 range_tree_add(addto, start, end - start);
770 return;
771 }
772
773 uint64_t overlap_start = MAX(rs_get_start(curr, removefrom),
774 start);
775 uint64_t overlap_end = MIN(rs_get_end(curr, removefrom),
776 end);
777 uint64_t overlap_size = overlap_end - overlap_start;
778 ASSERT3S(overlap_size, >, 0);
779 range_seg_max_t rs;
780 rs_copy(curr, &rs, removefrom);
781
782 range_tree_remove(removefrom, overlap_start, overlap_size);
783
784 if (start < overlap_start)
785 range_tree_add(addto, start, overlap_start - start);
786
787 start = overlap_end;
788 next = zfs_btree_find(&removefrom->rt_root, &rs, &where);
789 /*
790 * If we find something here, we only removed part of the
791 * curr segment. Either there's some left at the end
792 * because we've reached the end of the range we're removing,
793 * or there's some left at the start because we started
794 * partway through the range. Either way, we continue with
795 * the loop. If it's the former, we'll return at the start of
796 * the loop, and if it's the latter we'll see if there is more
797 * area to process.
798 */
799 if (next != NULL) {
800 ASSERT(start == end || start == rs_get_end(&rs,
801 removefrom));
802 }
803
804 next = zfs_btree_next(&removefrom->rt_root, &where, &where);
805 }
806 VERIFY3P(curr, ==, NULL);
807
808 if (start != end) {
809 VERIFY3U(start, <, end);
810 range_tree_add(addto, start, end - start);
811 } else {
812 VERIFY3U(start, ==, end);
813 }
814 }
815
816 /*
817 * For each entry in rt, if it exists in removefrom, remove it
818 * from removefrom. Otherwise, add it to addto.
819 */
820 void
range_tree_remove_xor_add(range_tree_t * rt,range_tree_t * removefrom,range_tree_t * addto)821 range_tree_remove_xor_add(range_tree_t *rt, range_tree_t *removefrom,
822 range_tree_t *addto)
823 {
824 zfs_btree_index_t where;
825 for (range_seg_t *rs = zfs_btree_first(&rt->rt_root, &where); rs;
826 rs = zfs_btree_next(&rt->rt_root, &where, &where)) {
827 range_tree_remove_xor_add_segment(rs_get_start(rs, rt),
828 rs_get_end(rs, rt), removefrom, addto);
829 }
830 }
831
832 uint64_t
range_tree_min(range_tree_t * rt)833 range_tree_min(range_tree_t *rt)
834 {
835 range_seg_t *rs = zfs_btree_first(&rt->rt_root, NULL);
836 return (rs != NULL ? rs_get_start(rs, rt) : 0);
837 }
838
839 uint64_t
range_tree_max(range_tree_t * rt)840 range_tree_max(range_tree_t *rt)
841 {
842 range_seg_t *rs = zfs_btree_last(&rt->rt_root, NULL);
843 return (rs != NULL ? rs_get_end(rs, rt) : 0);
844 }
845
846 uint64_t
range_tree_span(range_tree_t * rt)847 range_tree_span(range_tree_t *rt)
848 {
849 return (range_tree_max(rt) - range_tree_min(rt));
850 }
851