1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
24 */
25
26 #include <sys/spa.h>
27 #include <sys/spa_impl.h>
28 #include <sys/txg.h>
29 #include <sys/vdev_impl.h>
30 #include <sys/metaslab_impl.h>
31 #include <sys/dsl_synctask.h>
32 #include <sys/zap.h>
33 #include <sys/dmu_tx.h>
34 #include <sys/vdev_initialize.h>
35
36 /*
37 * Value that is written to disk during initialization.
38 */
39 #ifdef _ILP32
40 unsigned long zfs_initialize_value = 0xdeadbeefUL;
41 #else
42 unsigned long zfs_initialize_value = 0xdeadbeefdeadbeeeULL;
43 #endif
44
45 /* maximum number of I/Os outstanding per leaf vdev */
46 int zfs_initialize_limit = 1;
47
48 /* size of initializing writes; default 1MiB, see zfs_remove_max_segment */
49 unsigned long zfs_initialize_chunk_size = 1024 * 1024;
50
51 static boolean_t
vdev_initialize_should_stop(vdev_t * vd)52 vdev_initialize_should_stop(vdev_t *vd)
53 {
54 return (vd->vdev_initialize_exit_wanted || !vdev_writeable(vd) ||
55 vd->vdev_detached || vd->vdev_top->vdev_removing);
56 }
57
58 static void
vdev_initialize_zap_update_sync(void * arg,dmu_tx_t * tx)59 vdev_initialize_zap_update_sync(void *arg, dmu_tx_t *tx)
60 {
61 /*
62 * We pass in the guid instead of the vdev_t since the vdev may
63 * have been freed prior to the sync task being processed. This
64 * happens when a vdev is detached as we call spa_config_vdev_exit(),
65 * stop the initializing thread, schedule the sync task, and free
66 * the vdev. Later when the scheduled sync task is invoked, it would
67 * find that the vdev has been freed.
68 */
69 uint64_t guid = *(uint64_t *)arg;
70 uint64_t txg = dmu_tx_get_txg(tx);
71 kmem_free(arg, sizeof (uint64_t));
72
73 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
74 if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
75 return;
76
77 uint64_t last_offset = vd->vdev_initialize_offset[txg & TXG_MASK];
78 vd->vdev_initialize_offset[txg & TXG_MASK] = 0;
79
80 VERIFY(vd->vdev_leaf_zap != 0);
81
82 objset_t *mos = vd->vdev_spa->spa_meta_objset;
83
84 if (last_offset > 0) {
85 vd->vdev_initialize_last_offset = last_offset;
86 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
87 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
88 sizeof (last_offset), 1, &last_offset, tx));
89 }
90 if (vd->vdev_initialize_action_time > 0) {
91 uint64_t val = (uint64_t)vd->vdev_initialize_action_time;
92 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
93 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, sizeof (val),
94 1, &val, tx));
95 }
96
97 uint64_t initialize_state = vd->vdev_initialize_state;
98 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
99 VDEV_LEAF_ZAP_INITIALIZE_STATE, sizeof (initialize_state), 1,
100 &initialize_state, tx));
101 }
102
103 static void
vdev_initialize_zap_remove_sync(void * arg,dmu_tx_t * tx)104 vdev_initialize_zap_remove_sync(void *arg, dmu_tx_t *tx)
105 {
106 uint64_t guid = *(uint64_t *)arg;
107
108 kmem_free(arg, sizeof (uint64_t));
109
110 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
111 if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
112 return;
113
114 ASSERT3S(vd->vdev_initialize_state, ==, VDEV_INITIALIZE_NONE);
115 ASSERT3U(vd->vdev_leaf_zap, !=, 0);
116
117 vd->vdev_initialize_last_offset = 0;
118 vd->vdev_initialize_action_time = 0;
119
120 objset_t *mos = vd->vdev_spa->spa_meta_objset;
121 int error;
122
123 error = zap_remove(mos, vd->vdev_leaf_zap,
124 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET, tx);
125 VERIFY(error == 0 || error == ENOENT);
126
127 error = zap_remove(mos, vd->vdev_leaf_zap,
128 VDEV_LEAF_ZAP_INITIALIZE_STATE, tx);
129 VERIFY(error == 0 || error == ENOENT);
130
131 error = zap_remove(mos, vd->vdev_leaf_zap,
132 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, tx);
133 VERIFY(error == 0 || error == ENOENT);
134 }
135
136 static void
vdev_initialize_change_state(vdev_t * vd,vdev_initializing_state_t new_state)137 vdev_initialize_change_state(vdev_t *vd, vdev_initializing_state_t new_state)
138 {
139 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
140 spa_t *spa = vd->vdev_spa;
141
142 if (new_state == vd->vdev_initialize_state)
143 return;
144
145 /*
146 * Copy the vd's guid, this will be freed by the sync task.
147 */
148 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
149 *guid = vd->vdev_guid;
150
151 /*
152 * If we're suspending, then preserving the original start time.
153 */
154 if (vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) {
155 vd->vdev_initialize_action_time = gethrestime_sec();
156 }
157
158 vdev_initializing_state_t old_state = vd->vdev_initialize_state;
159 vd->vdev_initialize_state = new_state;
160
161 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
162 VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
163
164 if (new_state != VDEV_INITIALIZE_NONE) {
165 dsl_sync_task_nowait(spa_get_dsl(spa),
166 vdev_initialize_zap_update_sync, guid, tx);
167 } else {
168 dsl_sync_task_nowait(spa_get_dsl(spa),
169 vdev_initialize_zap_remove_sync, guid, tx);
170 }
171
172 switch (new_state) {
173 case VDEV_INITIALIZE_ACTIVE:
174 spa_history_log_internal(spa, "initialize", tx,
175 "vdev=%s activated", vd->vdev_path);
176 break;
177 case VDEV_INITIALIZE_SUSPENDED:
178 spa_history_log_internal(spa, "initialize", tx,
179 "vdev=%s suspended", vd->vdev_path);
180 break;
181 case VDEV_INITIALIZE_CANCELED:
182 if (old_state == VDEV_INITIALIZE_ACTIVE ||
183 old_state == VDEV_INITIALIZE_SUSPENDED)
184 spa_history_log_internal(spa, "initialize", tx,
185 "vdev=%s canceled", vd->vdev_path);
186 break;
187 case VDEV_INITIALIZE_COMPLETE:
188 spa_history_log_internal(spa, "initialize", tx,
189 "vdev=%s complete", vd->vdev_path);
190 break;
191 case VDEV_INITIALIZE_NONE:
192 spa_history_log_internal(spa, "uninitialize", tx,
193 "vdev=%s", vd->vdev_path);
194 break;
195 default:
196 panic("invalid state %llu", (unsigned long long)new_state);
197 }
198
199 dmu_tx_commit(tx);
200
201 if (new_state != VDEV_INITIALIZE_ACTIVE)
202 spa_notify_waiters(spa);
203 }
204
205 static void
vdev_initialize_cb(zio_t * zio)206 vdev_initialize_cb(zio_t *zio)
207 {
208 vdev_t *vd = zio->io_vd;
209 mutex_enter(&vd->vdev_initialize_io_lock);
210 if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
211 /*
212 * The I/O failed because the vdev was unavailable; roll the
213 * last offset back. (This works because spa_sync waits on
214 * spa_txg_zio before it runs sync tasks.)
215 */
216 uint64_t *off =
217 &vd->vdev_initialize_offset[zio->io_txg & TXG_MASK];
218 *off = MIN(*off, zio->io_offset);
219 } else {
220 /*
221 * Since initializing is best-effort, we ignore I/O errors and
222 * rely on vdev_probe to determine if the errors are more
223 * critical.
224 */
225 if (zio->io_error != 0)
226 vd->vdev_stat.vs_initialize_errors++;
227
228 vd->vdev_initialize_bytes_done += zio->io_orig_size;
229 }
230 ASSERT3U(vd->vdev_initialize_inflight, >, 0);
231 vd->vdev_initialize_inflight--;
232 cv_broadcast(&vd->vdev_initialize_io_cv);
233 mutex_exit(&vd->vdev_initialize_io_lock);
234
235 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
236 }
237
238 /* Takes care of physical writing and limiting # of concurrent ZIOs. */
239 static int
vdev_initialize_write(vdev_t * vd,uint64_t start,uint64_t size,abd_t * data)240 vdev_initialize_write(vdev_t *vd, uint64_t start, uint64_t size, abd_t *data)
241 {
242 spa_t *spa = vd->vdev_spa;
243
244 /* Limit inflight initializing I/Os */
245 mutex_enter(&vd->vdev_initialize_io_lock);
246 while (vd->vdev_initialize_inflight >= zfs_initialize_limit) {
247 cv_wait(&vd->vdev_initialize_io_cv,
248 &vd->vdev_initialize_io_lock);
249 }
250 vd->vdev_initialize_inflight++;
251 mutex_exit(&vd->vdev_initialize_io_lock);
252
253 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
254 VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
255 uint64_t txg = dmu_tx_get_txg(tx);
256
257 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
258 mutex_enter(&vd->vdev_initialize_lock);
259
260 if (vd->vdev_initialize_offset[txg & TXG_MASK] == 0) {
261 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
262 *guid = vd->vdev_guid;
263
264 /* This is the first write of this txg. */
265 dsl_sync_task_nowait(spa_get_dsl(spa),
266 vdev_initialize_zap_update_sync, guid, tx);
267 }
268
269 /*
270 * We know the vdev struct will still be around since all
271 * consumers of vdev_free must stop the initialization first.
272 */
273 if (vdev_initialize_should_stop(vd)) {
274 mutex_enter(&vd->vdev_initialize_io_lock);
275 ASSERT3U(vd->vdev_initialize_inflight, >, 0);
276 vd->vdev_initialize_inflight--;
277 mutex_exit(&vd->vdev_initialize_io_lock);
278 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
279 mutex_exit(&vd->vdev_initialize_lock);
280 dmu_tx_commit(tx);
281 return (SET_ERROR(EINTR));
282 }
283 mutex_exit(&vd->vdev_initialize_lock);
284
285 vd->vdev_initialize_offset[txg & TXG_MASK] = start + size;
286 zio_nowait(zio_write_phys(spa->spa_txg_zio[txg & TXG_MASK], vd, start,
287 size, data, ZIO_CHECKSUM_OFF, vdev_initialize_cb, NULL,
288 ZIO_PRIORITY_INITIALIZING, ZIO_FLAG_CANFAIL, B_FALSE));
289 /* vdev_initialize_cb releases SCL_STATE_ALL */
290
291 dmu_tx_commit(tx);
292
293 return (0);
294 }
295
296 /*
297 * Callback to fill each ABD chunk with zfs_initialize_value. len must be
298 * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD
299 * allocation will guarantee these for us.
300 */
301 static int
vdev_initialize_block_fill(void * buf,size_t len,void * unused)302 vdev_initialize_block_fill(void *buf, size_t len, void *unused)
303 {
304 (void) unused;
305
306 ASSERT0(len % sizeof (uint64_t));
307 #ifdef _ILP32
308 for (uint64_t i = 0; i < len; i += sizeof (uint32_t)) {
309 *(uint32_t *)((char *)(buf) + i) = zfs_initialize_value;
310 }
311 #else
312 for (uint64_t i = 0; i < len; i += sizeof (uint64_t)) {
313 *(uint64_t *)((char *)(buf) + i) = zfs_initialize_value;
314 }
315 #endif
316 return (0);
317 }
318
319 static abd_t *
vdev_initialize_block_alloc(void)320 vdev_initialize_block_alloc(void)
321 {
322 /* Allocate ABD for filler data */
323 abd_t *data = abd_alloc_for_io(zfs_initialize_chunk_size, B_FALSE);
324
325 ASSERT0(zfs_initialize_chunk_size % sizeof (uint64_t));
326 (void) abd_iterate_func(data, 0, zfs_initialize_chunk_size,
327 vdev_initialize_block_fill, NULL);
328
329 return (data);
330 }
331
332 static void
vdev_initialize_block_free(abd_t * data)333 vdev_initialize_block_free(abd_t *data)
334 {
335 abd_free(data);
336 }
337
338 static int
vdev_initialize_ranges(vdev_t * vd,abd_t * data)339 vdev_initialize_ranges(vdev_t *vd, abd_t *data)
340 {
341 range_tree_t *rt = vd->vdev_initialize_tree;
342 zfs_btree_t *bt = &rt->rt_root;
343 zfs_btree_index_t where;
344
345 for (range_seg_t *rs = zfs_btree_first(bt, &where); rs != NULL;
346 rs = zfs_btree_next(bt, &where, &where)) {
347 uint64_t size = rs_get_end(rs, rt) - rs_get_start(rs, rt);
348
349 /* Split range into legally-sized physical chunks */
350 uint64_t writes_required =
351 ((size - 1) / zfs_initialize_chunk_size) + 1;
352
353 for (uint64_t w = 0; w < writes_required; w++) {
354 int error;
355
356 error = vdev_initialize_write(vd,
357 VDEV_LABEL_START_SIZE + rs_get_start(rs, rt) +
358 (w * zfs_initialize_chunk_size),
359 MIN(size - (w * zfs_initialize_chunk_size),
360 zfs_initialize_chunk_size), data);
361 if (error != 0)
362 return (error);
363 }
364 }
365 return (0);
366 }
367
368 static void
vdev_initialize_xlate_last_rs_end(void * arg,range_seg64_t * physical_rs)369 vdev_initialize_xlate_last_rs_end(void *arg, range_seg64_t *physical_rs)
370 {
371 uint64_t *last_rs_end = (uint64_t *)arg;
372
373 if (physical_rs->rs_end > *last_rs_end)
374 *last_rs_end = physical_rs->rs_end;
375 }
376
377 static void
vdev_initialize_xlate_progress(void * arg,range_seg64_t * physical_rs)378 vdev_initialize_xlate_progress(void *arg, range_seg64_t *physical_rs)
379 {
380 vdev_t *vd = (vdev_t *)arg;
381
382 uint64_t size = physical_rs->rs_end - physical_rs->rs_start;
383 vd->vdev_initialize_bytes_est += size;
384
385 if (vd->vdev_initialize_last_offset > physical_rs->rs_end) {
386 vd->vdev_initialize_bytes_done += size;
387 } else if (vd->vdev_initialize_last_offset > physical_rs->rs_start &&
388 vd->vdev_initialize_last_offset < physical_rs->rs_end) {
389 vd->vdev_initialize_bytes_done +=
390 vd->vdev_initialize_last_offset - physical_rs->rs_start;
391 }
392 }
393
394 static void
vdev_initialize_calculate_progress(vdev_t * vd)395 vdev_initialize_calculate_progress(vdev_t *vd)
396 {
397 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
398 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
399 ASSERT(vd->vdev_leaf_zap != 0);
400
401 vd->vdev_initialize_bytes_est = 0;
402 vd->vdev_initialize_bytes_done = 0;
403
404 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
405 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
406 mutex_enter(&msp->ms_lock);
407
408 uint64_t ms_free = (msp->ms_size -
409 metaslab_allocated_space(msp)) /
410 vdev_get_ndisks(vd->vdev_top);
411
412 /*
413 * Convert the metaslab range to a physical range
414 * on our vdev. We use this to determine if we are
415 * in the middle of this metaslab range.
416 */
417 range_seg64_t logical_rs, physical_rs, remain_rs;
418 logical_rs.rs_start = msp->ms_start;
419 logical_rs.rs_end = msp->ms_start + msp->ms_size;
420
421 /* Metaslab space after this offset has not been initialized */
422 vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs);
423 if (vd->vdev_initialize_last_offset <= physical_rs.rs_start) {
424 vd->vdev_initialize_bytes_est += ms_free;
425 mutex_exit(&msp->ms_lock);
426 continue;
427 }
428
429 /* Metaslab space before this offset has been initialized */
430 uint64_t last_rs_end = physical_rs.rs_end;
431 if (!vdev_xlate_is_empty(&remain_rs)) {
432 vdev_xlate_walk(vd, &remain_rs,
433 vdev_initialize_xlate_last_rs_end, &last_rs_end);
434 }
435
436 if (vd->vdev_initialize_last_offset > last_rs_end) {
437 vd->vdev_initialize_bytes_done += ms_free;
438 vd->vdev_initialize_bytes_est += ms_free;
439 mutex_exit(&msp->ms_lock);
440 continue;
441 }
442
443 /*
444 * If we get here, we're in the middle of initializing this
445 * metaslab. Load it and walk the free tree for more accurate
446 * progress estimation.
447 */
448 VERIFY0(metaslab_load(msp));
449
450 zfs_btree_index_t where;
451 range_tree_t *rt = msp->ms_allocatable;
452 for (range_seg_t *rs =
453 zfs_btree_first(&rt->rt_root, &where); rs;
454 rs = zfs_btree_next(&rt->rt_root, &where,
455 &where)) {
456 logical_rs.rs_start = rs_get_start(rs, rt);
457 logical_rs.rs_end = rs_get_end(rs, rt);
458
459 vdev_xlate_walk(vd, &logical_rs,
460 vdev_initialize_xlate_progress, vd);
461 }
462 mutex_exit(&msp->ms_lock);
463 }
464 }
465
466 static int
vdev_initialize_load(vdev_t * vd)467 vdev_initialize_load(vdev_t *vd)
468 {
469 int err = 0;
470 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
471 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
472 ASSERT(vd->vdev_leaf_zap != 0);
473
474 if (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE ||
475 vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED) {
476 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
477 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
478 sizeof (vd->vdev_initialize_last_offset), 1,
479 &vd->vdev_initialize_last_offset);
480 if (err == ENOENT) {
481 vd->vdev_initialize_last_offset = 0;
482 err = 0;
483 }
484 }
485
486 vdev_initialize_calculate_progress(vd);
487 return (err);
488 }
489
490 static void
vdev_initialize_xlate_range_add(void * arg,range_seg64_t * physical_rs)491 vdev_initialize_xlate_range_add(void *arg, range_seg64_t *physical_rs)
492 {
493 vdev_t *vd = arg;
494
495 /* Only add segments that we have not visited yet */
496 if (physical_rs->rs_end <= vd->vdev_initialize_last_offset)
497 return;
498
499 /* Pick up where we left off mid-range. */
500 if (vd->vdev_initialize_last_offset > physical_rs->rs_start) {
501 zfs_dbgmsg("range write: vd %s changed (%llu, %llu) to "
502 "(%llu, %llu)", vd->vdev_path,
503 (u_longlong_t)physical_rs->rs_start,
504 (u_longlong_t)physical_rs->rs_end,
505 (u_longlong_t)vd->vdev_initialize_last_offset,
506 (u_longlong_t)physical_rs->rs_end);
507 ASSERT3U(physical_rs->rs_end, >,
508 vd->vdev_initialize_last_offset);
509 physical_rs->rs_start = vd->vdev_initialize_last_offset;
510 }
511
512 ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start);
513
514 range_tree_add(vd->vdev_initialize_tree, physical_rs->rs_start,
515 physical_rs->rs_end - physical_rs->rs_start);
516 }
517
518 /*
519 * Convert the logical range into a physical range and add it to our
520 * avl tree.
521 */
522 static void
vdev_initialize_range_add(void * arg,uint64_t start,uint64_t size)523 vdev_initialize_range_add(void *arg, uint64_t start, uint64_t size)
524 {
525 vdev_t *vd = arg;
526 range_seg64_t logical_rs;
527 logical_rs.rs_start = start;
528 logical_rs.rs_end = start + size;
529
530 ASSERT(vd->vdev_ops->vdev_op_leaf);
531 vdev_xlate_walk(vd, &logical_rs, vdev_initialize_xlate_range_add, arg);
532 }
533
534 static void
vdev_initialize_thread(void * arg)535 vdev_initialize_thread(void *arg)
536 {
537 vdev_t *vd = arg;
538 spa_t *spa = vd->vdev_spa;
539 int error = 0;
540 uint64_t ms_count = 0;
541
542 ASSERT(vdev_is_concrete(vd));
543 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
544
545 vd->vdev_initialize_last_offset = 0;
546 VERIFY0(vdev_initialize_load(vd));
547
548 abd_t *deadbeef = vdev_initialize_block_alloc();
549
550 vd->vdev_initialize_tree = range_tree_create(NULL, RANGE_SEG64, NULL,
551 0, 0);
552
553 for (uint64_t i = 0; !vd->vdev_detached &&
554 i < vd->vdev_top->vdev_ms_count; i++) {
555 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
556 boolean_t unload_when_done = B_FALSE;
557
558 /*
559 * If we've expanded the top-level vdev or it's our
560 * first pass, calculate our progress.
561 */
562 if (vd->vdev_top->vdev_ms_count != ms_count) {
563 vdev_initialize_calculate_progress(vd);
564 ms_count = vd->vdev_top->vdev_ms_count;
565 }
566
567 spa_config_exit(spa, SCL_CONFIG, FTAG);
568 metaslab_disable(msp);
569 mutex_enter(&msp->ms_lock);
570 if (!msp->ms_loaded && !msp->ms_loading)
571 unload_when_done = B_TRUE;
572 VERIFY0(metaslab_load(msp));
573
574 range_tree_walk(msp->ms_allocatable, vdev_initialize_range_add,
575 vd);
576 mutex_exit(&msp->ms_lock);
577
578 error = vdev_initialize_ranges(vd, deadbeef);
579 metaslab_enable(msp, B_TRUE, unload_when_done);
580 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
581
582 range_tree_vacate(vd->vdev_initialize_tree, NULL, NULL);
583 if (error != 0)
584 break;
585 }
586
587 spa_config_exit(spa, SCL_CONFIG, FTAG);
588 mutex_enter(&vd->vdev_initialize_io_lock);
589 while (vd->vdev_initialize_inflight > 0) {
590 cv_wait(&vd->vdev_initialize_io_cv,
591 &vd->vdev_initialize_io_lock);
592 }
593 mutex_exit(&vd->vdev_initialize_io_lock);
594
595 range_tree_destroy(vd->vdev_initialize_tree);
596 vdev_initialize_block_free(deadbeef);
597 vd->vdev_initialize_tree = NULL;
598
599 mutex_enter(&vd->vdev_initialize_lock);
600 if (!vd->vdev_initialize_exit_wanted) {
601 if (vdev_writeable(vd)) {
602 vdev_initialize_change_state(vd,
603 VDEV_INITIALIZE_COMPLETE);
604 } else if (vd->vdev_faulted) {
605 vdev_initialize_change_state(vd,
606 VDEV_INITIALIZE_CANCELED);
607 }
608 }
609 ASSERT(vd->vdev_initialize_thread != NULL ||
610 vd->vdev_initialize_inflight == 0);
611
612 /*
613 * Drop the vdev_initialize_lock while we sync out the
614 * txg since it's possible that a device might be trying to
615 * come online and must check to see if it needs to restart an
616 * initialization. That thread will be holding the spa_config_lock
617 * which would prevent the txg_wait_synced from completing.
618 */
619 mutex_exit(&vd->vdev_initialize_lock);
620 txg_wait_synced(spa_get_dsl(spa), 0);
621 mutex_enter(&vd->vdev_initialize_lock);
622
623 vd->vdev_initialize_thread = NULL;
624 cv_broadcast(&vd->vdev_initialize_cv);
625 mutex_exit(&vd->vdev_initialize_lock);
626
627 thread_exit();
628 }
629
630 /*
631 * Initiates a device. Caller must hold vdev_initialize_lock.
632 * Device must be a leaf and not already be initializing.
633 */
634 void
vdev_initialize(vdev_t * vd)635 vdev_initialize(vdev_t *vd)
636 {
637 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
638 ASSERT(vd->vdev_ops->vdev_op_leaf);
639 ASSERT(vdev_is_concrete(vd));
640 ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
641 ASSERT(!vd->vdev_detached);
642 ASSERT(!vd->vdev_initialize_exit_wanted);
643 ASSERT(!vd->vdev_top->vdev_removing);
644
645 vdev_initialize_change_state(vd, VDEV_INITIALIZE_ACTIVE);
646 vd->vdev_initialize_thread = thread_create(NULL, 0,
647 vdev_initialize_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
648 }
649
650 /*
651 * Uninitializes a device. Caller must hold vdev_initialize_lock.
652 * Device must be a leaf and not already be initializing.
653 */
654 void
vdev_uninitialize(vdev_t * vd)655 vdev_uninitialize(vdev_t *vd)
656 {
657 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
658 ASSERT(vd->vdev_ops->vdev_op_leaf);
659 ASSERT(vdev_is_concrete(vd));
660 ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
661 ASSERT(!vd->vdev_detached);
662 ASSERT(!vd->vdev_initialize_exit_wanted);
663 ASSERT(!vd->vdev_top->vdev_removing);
664
665 vdev_initialize_change_state(vd, VDEV_INITIALIZE_NONE);
666 }
667
668 /*
669 * Wait for the initialize thread to be terminated (cancelled or stopped).
670 */
671 static void
vdev_initialize_stop_wait_impl(vdev_t * vd)672 vdev_initialize_stop_wait_impl(vdev_t *vd)
673 {
674 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
675
676 while (vd->vdev_initialize_thread != NULL)
677 cv_wait(&vd->vdev_initialize_cv, &vd->vdev_initialize_lock);
678
679 ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
680 vd->vdev_initialize_exit_wanted = B_FALSE;
681 }
682
683 /*
684 * Wait for vdev initialize threads which were either to cleanly exit.
685 */
686 void
vdev_initialize_stop_wait(spa_t * spa,list_t * vd_list)687 vdev_initialize_stop_wait(spa_t *spa, list_t *vd_list)
688 {
689 (void) spa;
690 vdev_t *vd;
691
692 ASSERT(MUTEX_HELD(&spa_namespace_lock));
693
694 while ((vd = list_remove_head(vd_list)) != NULL) {
695 mutex_enter(&vd->vdev_initialize_lock);
696 vdev_initialize_stop_wait_impl(vd);
697 mutex_exit(&vd->vdev_initialize_lock);
698 }
699 }
700
701 /*
702 * Stop initializing a device, with the resultant initializing state being
703 * tgt_state. For blocking behavior pass NULL for vd_list. Otherwise, when
704 * a list_t is provided the stopping vdev is inserted in to the list. Callers
705 * are then required to call vdev_initialize_stop_wait() to block for all the
706 * initialization threads to exit. The caller must hold vdev_initialize_lock
707 * and must not be writing to the spa config, as the initializing thread may
708 * try to enter the config as a reader before exiting.
709 */
710 void
vdev_initialize_stop(vdev_t * vd,vdev_initializing_state_t tgt_state,list_t * vd_list)711 vdev_initialize_stop(vdev_t *vd, vdev_initializing_state_t tgt_state,
712 list_t *vd_list)
713 {
714 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
715 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
716 ASSERT(vd->vdev_ops->vdev_op_leaf);
717 ASSERT(vdev_is_concrete(vd));
718
719 /*
720 * Allow cancel requests to proceed even if the initialize thread
721 * has stopped.
722 */
723 if (vd->vdev_initialize_thread == NULL &&
724 tgt_state != VDEV_INITIALIZE_CANCELED) {
725 return;
726 }
727
728 vdev_initialize_change_state(vd, tgt_state);
729 vd->vdev_initialize_exit_wanted = B_TRUE;
730
731 if (vd_list == NULL) {
732 vdev_initialize_stop_wait_impl(vd);
733 } else {
734 ASSERT(MUTEX_HELD(&spa_namespace_lock));
735 list_insert_tail(vd_list, vd);
736 }
737 }
738
739 static void
vdev_initialize_stop_all_impl(vdev_t * vd,vdev_initializing_state_t tgt_state,list_t * vd_list)740 vdev_initialize_stop_all_impl(vdev_t *vd, vdev_initializing_state_t tgt_state,
741 list_t *vd_list)
742 {
743 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
744 mutex_enter(&vd->vdev_initialize_lock);
745 vdev_initialize_stop(vd, tgt_state, vd_list);
746 mutex_exit(&vd->vdev_initialize_lock);
747 return;
748 }
749
750 for (uint64_t i = 0; i < vd->vdev_children; i++) {
751 vdev_initialize_stop_all_impl(vd->vdev_child[i], tgt_state,
752 vd_list);
753 }
754 }
755
756 /*
757 * Convenience function to stop initializing of a vdev tree and set all
758 * initialize thread pointers to NULL.
759 */
760 void
vdev_initialize_stop_all(vdev_t * vd,vdev_initializing_state_t tgt_state)761 vdev_initialize_stop_all(vdev_t *vd, vdev_initializing_state_t tgt_state)
762 {
763 spa_t *spa = vd->vdev_spa;
764 list_t vd_list;
765
766 ASSERT(MUTEX_HELD(&spa_namespace_lock));
767
768 list_create(&vd_list, sizeof (vdev_t),
769 offsetof(vdev_t, vdev_initialize_node));
770
771 vdev_initialize_stop_all_impl(vd, tgt_state, &vd_list);
772 vdev_initialize_stop_wait(spa, &vd_list);
773
774 if (vd->vdev_spa->spa_sync_on) {
775 /* Make sure that our state has been synced to disk */
776 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
777 }
778
779 list_destroy(&vd_list);
780 }
781
782 void
vdev_initialize_restart(vdev_t * vd)783 vdev_initialize_restart(vdev_t *vd)
784 {
785 ASSERT(MUTEX_HELD(&spa_namespace_lock));
786 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
787
788 if (vd->vdev_leaf_zap != 0) {
789 mutex_enter(&vd->vdev_initialize_lock);
790 uint64_t initialize_state = VDEV_INITIALIZE_NONE;
791 int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
792 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_STATE,
793 sizeof (initialize_state), 1, &initialize_state);
794 ASSERT(err == 0 || err == ENOENT);
795 vd->vdev_initialize_state = initialize_state;
796
797 uint64_t timestamp = 0;
798 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
799 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME,
800 sizeof (timestamp), 1, ×tamp);
801 ASSERT(err == 0 || err == ENOENT);
802 vd->vdev_initialize_action_time = timestamp;
803
804 if (vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
805 vd->vdev_offline) {
806 /* load progress for reporting, but don't resume */
807 VERIFY0(vdev_initialize_load(vd));
808 } else if (vd->vdev_initialize_state ==
809 VDEV_INITIALIZE_ACTIVE && vdev_writeable(vd) &&
810 !vd->vdev_top->vdev_removing &&
811 vd->vdev_initialize_thread == NULL) {
812 vdev_initialize(vd);
813 }
814
815 mutex_exit(&vd->vdev_initialize_lock);
816 }
817
818 for (uint64_t i = 0; i < vd->vdev_children; i++) {
819 vdev_initialize_restart(vd->vdev_child[i]);
820 }
821 }
822
823 EXPORT_SYMBOL(vdev_initialize);
824 EXPORT_SYMBOL(vdev_uninitialize);
825 EXPORT_SYMBOL(vdev_initialize_stop);
826 EXPORT_SYMBOL(vdev_initialize_stop_all);
827 EXPORT_SYMBOL(vdev_initialize_stop_wait);
828 EXPORT_SYMBOL(vdev_initialize_restart);
829
830 /* BEGIN CSTYLED */
831 ZFS_MODULE_PARAM(zfs, zfs_, initialize_value, ULONG, ZMOD_RW,
832 "Value written during zpool initialize");
833
834 ZFS_MODULE_PARAM(zfs, zfs_, initialize_chunk_size, ULONG, ZMOD_RW,
835 "Size in bytes of writes by zpool initialize");
836 /* END CSTYLED */
837