Lines Matching refs:rr
139 vdev_raidz_row_free(raidz_row_t *rr) in vdev_raidz_row_free() argument
141 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_row_free()
142 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_row_free()
150 if (rr->rr_abd_empty != NULL) in vdev_raidz_row_free()
151 abd_free(rr->rr_abd_empty); in vdev_raidz_row_free()
153 kmem_free(rr, offsetof(raidz_row_t, rr_col[rr->rr_scols])); in vdev_raidz_row_free()
183 raidz_row_t *rr = rm->rm_row[0]; in vdev_raidz_map_alloc_write() local
192 if (rm->rm_skipstart < rr->rr_firstdatacol) { in vdev_raidz_map_alloc_write()
195 } else if (rr->rr_scols < (rm->rm_skipstart + rm->rm_nskip)) { in vdev_raidz_map_alloc_write()
197 (rm->rm_skipstart + rm->rm_nskip) % rr->rr_scols; in vdev_raidz_map_alloc_write()
204 int skipped = rr->rr_scols - rr->rr_cols; in vdev_raidz_map_alloc_write()
207 for (c = 0; c < rr->rr_firstdatacol; c++) { in vdev_raidz_map_alloc_write()
208 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_write()
229 for (off = 0; c < rr->rr_cols; c++) { in vdev_raidz_map_alloc_write()
230 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_write()
268 raidz_row_t *rr = rm->rm_row[0]; in vdev_raidz_map_alloc_read() local
273 for (c = 0; c < rr->rr_firstdatacol; c++) in vdev_raidz_map_alloc_read()
274 rr->rr_col[c].rc_abd = in vdev_raidz_map_alloc_read()
275 abd_alloc_linear(rr->rr_col[c].rc_size, B_FALSE); in vdev_raidz_map_alloc_read()
277 for (uint64_t off = 0; c < rr->rr_cols; c++) { in vdev_raidz_map_alloc_read()
278 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_read()
296 raidz_row_t *rr; in vdev_raidz_map_alloc() local
347 rr = kmem_alloc(offsetof(raidz_row_t, rr_col[scols]), KM_SLEEP); in vdev_raidz_map_alloc()
348 rm->rm_row[0] = rr; in vdev_raidz_map_alloc()
350 rr->rr_cols = acols; in vdev_raidz_map_alloc()
351 rr->rr_scols = scols; in vdev_raidz_map_alloc()
352 rr->rr_bigcols = bc; in vdev_raidz_map_alloc()
353 rr->rr_missingdata = 0; in vdev_raidz_map_alloc()
354 rr->rr_missingparity = 0; in vdev_raidz_map_alloc()
355 rr->rr_firstdatacol = nparity; in vdev_raidz_map_alloc()
356 rr->rr_abd_empty = NULL; in vdev_raidz_map_alloc()
357 rr->rr_nempty = 0; in vdev_raidz_map_alloc()
359 rr->rr_offset = zio->io_offset; in vdev_raidz_map_alloc()
360 rr->rr_size = zio->io_size; in vdev_raidz_map_alloc()
366 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc()
418 ASSERT(rr->rr_cols >= 2); in vdev_raidz_map_alloc()
419 ASSERT(rr->rr_col[0].rc_size == rr->rr_col[1].rc_size); in vdev_raidz_map_alloc()
421 if (rr->rr_firstdatacol == 1 && (zio->io_offset & (1ULL << 20))) { in vdev_raidz_map_alloc()
422 devidx = rr->rr_col[0].rc_devidx; in vdev_raidz_map_alloc()
423 o = rr->rr_col[0].rc_offset; in vdev_raidz_map_alloc()
424 rr->rr_col[0].rc_devidx = rr->rr_col[1].rc_devidx; in vdev_raidz_map_alloc()
425 rr->rr_col[0].rc_offset = rr->rr_col[1].rc_offset; in vdev_raidz_map_alloc()
426 rr->rr_col[1].rc_devidx = devidx; in vdev_raidz_map_alloc()
427 rr->rr_col[1].rc_offset = o; in vdev_raidz_map_alloc()
507 vdev_raidz_generate_parity_p(raidz_row_t *rr) in vdev_raidz_generate_parity_p() argument
509 uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_generate_parity_p()
511 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_generate_parity_p()
512 abd_t *src = rr->rr_col[c].rc_abd; in vdev_raidz_generate_parity_p()
514 if (c == rr->rr_firstdatacol) { in vdev_raidz_generate_parity_p()
515 abd_copy_to_buf(p, src, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_p()
518 (void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size, in vdev_raidz_generate_parity_p()
525 vdev_raidz_generate_parity_pq(raidz_row_t *rr) in vdev_raidz_generate_parity_pq() argument
527 uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_generate_parity_pq()
528 uint64_t *q = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_generate_parity_pq()
529 uint64_t pcnt = rr->rr_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pq()
530 ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size == in vdev_raidz_generate_parity_pq()
531 rr->rr_col[VDEV_RAIDZ_Q].rc_size); in vdev_raidz_generate_parity_pq()
533 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_generate_parity_pq()
534 abd_t *src = rr->rr_col[c].rc_abd; in vdev_raidz_generate_parity_pq()
536 uint64_t ccnt = rr->rr_col[c].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pq()
538 if (c == rr->rr_firstdatacol) { in vdev_raidz_generate_parity_pq()
540 abd_copy_to_buf(p, src, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pq()
541 (void) memcpy(q, p, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pq()
551 (void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size, in vdev_raidz_generate_parity_pq()
567 vdev_raidz_generate_parity_pqr(raidz_row_t *rr) in vdev_raidz_generate_parity_pqr() argument
569 uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_generate_parity_pqr()
570 uint64_t *q = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_generate_parity_pqr()
571 uint64_t *r = abd_to_buf(rr->rr_col[VDEV_RAIDZ_R].rc_abd); in vdev_raidz_generate_parity_pqr()
572 uint64_t pcnt = rr->rr_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pqr()
573 ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size == in vdev_raidz_generate_parity_pqr()
574 rr->rr_col[VDEV_RAIDZ_Q].rc_size); in vdev_raidz_generate_parity_pqr()
575 ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size == in vdev_raidz_generate_parity_pqr()
576 rr->rr_col[VDEV_RAIDZ_R].rc_size); in vdev_raidz_generate_parity_pqr()
578 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_generate_parity_pqr()
579 abd_t *src = rr->rr_col[c].rc_abd; in vdev_raidz_generate_parity_pqr()
581 uint64_t ccnt = rr->rr_col[c].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pqr()
583 if (c == rr->rr_firstdatacol) { in vdev_raidz_generate_parity_pqr()
585 abd_copy_to_buf(p, src, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
586 (void) memcpy(q, p, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
587 (void) memcpy(r, p, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
598 (void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size, in vdev_raidz_generate_parity_pqr()
619 vdev_raidz_generate_parity_row(raidz_map_t *rm, raidz_row_t *rr) in vdev_raidz_generate_parity_row() argument
621 ASSERT3U(rr->rr_cols, !=, 0); in vdev_raidz_generate_parity_row()
624 if (vdev_raidz_math_generate(rm, rr) != RAIDZ_ORIGINAL_IMPL) in vdev_raidz_generate_parity_row()
627 switch (rr->rr_firstdatacol) { in vdev_raidz_generate_parity_row()
629 vdev_raidz_generate_parity_p(rr); in vdev_raidz_generate_parity_row()
632 vdev_raidz_generate_parity_pq(rr); in vdev_raidz_generate_parity_row()
635 vdev_raidz_generate_parity_pqr(rr); in vdev_raidz_generate_parity_row()
646 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_generate_parity() local
647 vdev_raidz_generate_parity_row(rm, rr); in vdev_raidz_generate_parity()
768 vdev_raidz_reconstruct_p(raidz_row_t *rr, int *tgts, int ntgts) in vdev_raidz_reconstruct_p() argument
774 ASSERT3U(x, >=, rr->rr_firstdatacol); in vdev_raidz_reconstruct_p()
775 ASSERT3U(x, <, rr->rr_cols); in vdev_raidz_reconstruct_p()
777 ASSERT3U(rr->rr_col[x].rc_size, <=, rr->rr_col[VDEV_RAIDZ_P].rc_size); in vdev_raidz_reconstruct_p()
779 src = rr->rr_col[VDEV_RAIDZ_P].rc_abd; in vdev_raidz_reconstruct_p()
780 dst = rr->rr_col[x].rc_abd; in vdev_raidz_reconstruct_p()
782 abd_copy_from_buf(dst, abd_to_buf(src), rr->rr_col[x].rc_size); in vdev_raidz_reconstruct_p()
784 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_p()
785 uint64_t size = MIN(rr->rr_col[x].rc_size, in vdev_raidz_reconstruct_p()
786 rr->rr_col[c].rc_size); in vdev_raidz_reconstruct_p()
788 src = rr->rr_col[c].rc_abd; in vdev_raidz_reconstruct_p()
799 vdev_raidz_reconstruct_q(raidz_row_t *rr, int *tgts, int ntgts) in vdev_raidz_reconstruct_q() argument
807 ASSERT(rr->rr_col[x].rc_size <= rr->rr_col[VDEV_RAIDZ_Q].rc_size); in vdev_raidz_reconstruct_q()
809 for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_q()
810 uint64_t size = (c == x) ? 0 : MIN(rr->rr_col[x].rc_size, in vdev_raidz_reconstruct_q()
811 rr->rr_col[c].rc_size); in vdev_raidz_reconstruct_q()
813 src = rr->rr_col[c].rc_abd; in vdev_raidz_reconstruct_q()
814 dst = rr->rr_col[x].rc_abd; in vdev_raidz_reconstruct_q()
816 if (c == rr->rr_firstdatacol) { in vdev_raidz_reconstruct_q()
818 if (rr->rr_col[x].rc_size > size) { in vdev_raidz_reconstruct_q()
820 rr->rr_col[x].rc_size - size); in vdev_raidz_reconstruct_q()
823 ASSERT3U(size, <=, rr->rr_col[x].rc_size); in vdev_raidz_reconstruct_q()
827 size, rr->rr_col[x].rc_size - size, in vdev_raidz_reconstruct_q()
832 src = rr->rr_col[VDEV_RAIDZ_Q].rc_abd; in vdev_raidz_reconstruct_q()
833 dst = rr->rr_col[x].rc_abd; in vdev_raidz_reconstruct_q()
834 exp = 255 - (rr->rr_cols - 1 - x); in vdev_raidz_reconstruct_q()
837 (void) abd_iterate_func(dst, 0, rr->rr_col[x].rc_size, in vdev_raidz_reconstruct_q()
842 vdev_raidz_reconstruct_pq(raidz_row_t *rr, int *tgts, int ntgts) in vdev_raidz_reconstruct_pq() argument
853 ASSERT(x >= rr->rr_firstdatacol); in vdev_raidz_reconstruct_pq()
854 ASSERT(y < rr->rr_cols); in vdev_raidz_reconstruct_pq()
856 ASSERT(rr->rr_col[x].rc_size >= rr->rr_col[y].rc_size); in vdev_raidz_reconstruct_pq()
865 pdata = rr->rr_col[VDEV_RAIDZ_P].rc_abd; in vdev_raidz_reconstruct_pq()
866 qdata = rr->rr_col[VDEV_RAIDZ_Q].rc_abd; in vdev_raidz_reconstruct_pq()
867 xsize = rr->rr_col[x].rc_size; in vdev_raidz_reconstruct_pq()
868 ysize = rr->rr_col[y].rc_size; in vdev_raidz_reconstruct_pq()
870 rr->rr_col[VDEV_RAIDZ_P].rc_abd = in vdev_raidz_reconstruct_pq()
871 abd_alloc_linear(rr->rr_col[VDEV_RAIDZ_P].rc_size, B_TRUE); in vdev_raidz_reconstruct_pq()
872 rr->rr_col[VDEV_RAIDZ_Q].rc_abd = in vdev_raidz_reconstruct_pq()
873 abd_alloc_linear(rr->rr_col[VDEV_RAIDZ_Q].rc_size, B_TRUE); in vdev_raidz_reconstruct_pq()
874 rr->rr_col[x].rc_size = 0; in vdev_raidz_reconstruct_pq()
875 rr->rr_col[y].rc_size = 0; in vdev_raidz_reconstruct_pq()
877 vdev_raidz_generate_parity_pq(rr); in vdev_raidz_reconstruct_pq()
879 rr->rr_col[x].rc_size = xsize; in vdev_raidz_reconstruct_pq()
880 rr->rr_col[y].rc_size = ysize; in vdev_raidz_reconstruct_pq()
884 pxy = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_reconstruct_pq()
885 qxy = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_reconstruct_pq()
886 xd = rr->rr_col[x].rc_abd; in vdev_raidz_reconstruct_pq()
887 yd = rr->rr_col[y].rc_abd; in vdev_raidz_reconstruct_pq()
905 b = vdev_raidz_pow2[255 - (rr->rr_cols - 1 - x)]; in vdev_raidz_reconstruct_pq()
919 abd_free(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_reconstruct_pq()
920 abd_free(rr->rr_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_reconstruct_pq()
925 rr->rr_col[VDEV_RAIDZ_P].rc_abd = pdata; in vdev_raidz_reconstruct_pq()
926 rr->rr_col[VDEV_RAIDZ_Q].rc_abd = qdata; in vdev_raidz_reconstruct_pq()
1082 vdev_raidz_matrix_init(raidz_row_t *rr, int n, int nmap, int *map, in vdev_raidz_matrix_init() argument
1088 ASSERT(n == rr->rr_cols - rr->rr_firstdatacol); in vdev_raidz_matrix_init()
1112 vdev_raidz_matrix_invert(raidz_row_t *rr, int n, int nmissing, int *missing, in vdev_raidz_matrix_invert() argument
1124 ASSERT3S(used[i], <, rr->rr_firstdatacol); in vdev_raidz_matrix_invert()
1127 ASSERT3S(used[i], >=, rr->rr_firstdatacol); in vdev_raidz_matrix_invert()
1144 ASSERT3U(used[j], >=, rr->rr_firstdatacol); in vdev_raidz_matrix_invert()
1145 jj = used[j] - rr->rr_firstdatacol; in vdev_raidz_matrix_invert()
1206 vdev_raidz_matrix_reconstruct(raidz_row_t *rr, int n, int nmissing, in vdev_raidz_matrix_reconstruct() argument
1238 ASSERT3U(c, <, rr->rr_cols); in vdev_raidz_matrix_reconstruct()
1240 ccount = rr->rr_col[c].rc_size; in vdev_raidz_matrix_reconstruct()
1241 ASSERT(ccount >= rr->rr_col[missing[0]].rc_size || i > 0); in vdev_raidz_matrix_reconstruct()
1244 src = abd_to_buf(rr->rr_col[c].rc_abd); in vdev_raidz_matrix_reconstruct()
1246 cc = missing[j] + rr->rr_firstdatacol; in vdev_raidz_matrix_reconstruct()
1247 ASSERT3U(cc, >=, rr->rr_firstdatacol); in vdev_raidz_matrix_reconstruct()
1248 ASSERT3U(cc, <, rr->rr_cols); in vdev_raidz_matrix_reconstruct()
1251 dcount[j] = rr->rr_col[cc].rc_size; in vdev_raidz_matrix_reconstruct()
1253 dst[j] = abd_to_buf(rr->rr_col[cc].rc_abd); in vdev_raidz_matrix_reconstruct()
1284 vdev_raidz_reconstruct_general(raidz_row_t *rr, int *tgts, int ntgts) in vdev_raidz_reconstruct_general() argument
1303 for (i = rr->rr_firstdatacol; i < rr->rr_cols; i++) { in vdev_raidz_reconstruct_general()
1304 if (!abd_is_linear(rr->rr_col[i].rc_abd)) { in vdev_raidz_reconstruct_general()
1305 bufs = kmem_alloc(rr->rr_cols * sizeof (abd_t *), in vdev_raidz_reconstruct_general()
1308 for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_general()
1309 raidz_col_t *col = &rr->rr_col[c]; in vdev_raidz_reconstruct_general()
1324 n = rr->rr_cols - rr->rr_firstdatacol; in vdev_raidz_reconstruct_general()
1331 if (tgts[t] >= rr->rr_firstdatacol) { in vdev_raidz_reconstruct_general()
1333 tgts[t] - rr->rr_firstdatacol; in vdev_raidz_reconstruct_general()
1343 ASSERT(c < rr->rr_firstdatacol); in vdev_raidz_reconstruct_general()
1373 for (tt = 0, c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_general()
1375 c == missing_rows[tt] + rr->rr_firstdatacol) { in vdev_raidz_reconstruct_general()
1388 vdev_raidz_matrix_init(rr, n, nmissing_rows, parity_map, rows); in vdev_raidz_reconstruct_general()
1393 vdev_raidz_matrix_invert(rr, n, nmissing_rows, missing_rows, rows, in vdev_raidz_reconstruct_general()
1399 vdev_raidz_matrix_reconstruct(rr, n, nmissing_rows, missing_rows, in vdev_raidz_reconstruct_general()
1408 for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_general()
1409 raidz_col_t *col = &rr->rr_col[c]; in vdev_raidz_reconstruct_general()
1417 kmem_free(bufs, rr->rr_cols * sizeof (abd_t *)); in vdev_raidz_reconstruct_general()
1422 vdev_raidz_reconstruct_row(raidz_map_t *rm, raidz_row_t *rr, in vdev_raidz_reconstruct_row() argument
1431 nbadparity = rr->rr_firstdatacol; in vdev_raidz_reconstruct_row()
1432 nbaddata = rr->rr_cols - nbadparity; in vdev_raidz_reconstruct_row()
1434 for (i = 0, c = 0; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_row()
1435 if (c < rr->rr_firstdatacol) in vdev_raidz_reconstruct_row()
1441 } else if (rr->rr_col[c].rc_error != 0) { in vdev_raidz_reconstruct_row()
1443 } else if (c >= rr->rr_firstdatacol) { in vdev_raidz_reconstruct_row()
1458 ret = vdev_raidz_math_reconstruct(rm, rr, parity_valid, dt, nbaddata); in vdev_raidz_reconstruct_row()
1468 vdev_raidz_reconstruct_p(rr, dt, 1); in vdev_raidz_reconstruct_row()
1472 ASSERT(rr->rr_firstdatacol > 1); in vdev_raidz_reconstruct_row()
1475 vdev_raidz_reconstruct_q(rr, dt, 1); in vdev_raidz_reconstruct_row()
1479 ASSERT(rr->rr_firstdatacol > 2); in vdev_raidz_reconstruct_row()
1483 ASSERT(rr->rr_firstdatacol > 1); in vdev_raidz_reconstruct_row()
1487 vdev_raidz_reconstruct_pq(rr, dt, 2); in vdev_raidz_reconstruct_row()
1491 ASSERT(rr->rr_firstdatacol > 2); in vdev_raidz_reconstruct_row()
1496 vdev_raidz_reconstruct_general(rr, tgts, ntgts); in vdev_raidz_reconstruct_row()
1600 vdev_raidz_io_verify(vdev_t *vd, raidz_row_t *rr, int col) in vdev_raidz_io_verify() argument
1606 logical_rs.rs_start = rr->rr_offset; in vdev_raidz_io_verify()
1608 vdev_raidz_asize(vd, rr->rr_size); in vdev_raidz_io_verify()
1610 raidz_col_t *rc = &rr->rr_col[col]; in vdev_raidz_io_verify()
1633 vdev_raidz_io_start_write(zio_t *zio, raidz_row_t *rr, uint64_t ashift) in vdev_raidz_io_start_write() argument
1638 vdev_raidz_generate_parity_row(rm, rr); in vdev_raidz_io_start_write()
1640 for (int c = 0; c < rr->rr_scols; c++) { in vdev_raidz_io_start_write()
1641 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_start_write()
1645 vdev_raidz_io_verify(vd, rr, c); in vdev_raidz_io_start_write()
1669 vdev_raidz_io_start_read(zio_t *zio, raidz_row_t *rr) in vdev_raidz_io_start_read() argument
1677 for (int c = rr->rr_cols - 1; c >= 0; c--) { in vdev_raidz_io_start_read()
1678 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_start_read()
1683 if (c >= rr->rr_firstdatacol) in vdev_raidz_io_start_read()
1684 rr->rr_missingdata++; in vdev_raidz_io_start_read()
1686 rr->rr_missingparity++; in vdev_raidz_io_start_read()
1693 if (c >= rr->rr_firstdatacol) in vdev_raidz_io_start_read()
1694 rr->rr_missingdata++; in vdev_raidz_io_start_read()
1696 rr->rr_missingparity++; in vdev_raidz_io_start_read()
1701 if (c >= rr->rr_firstdatacol || rr->rr_missingdata > 0 || in vdev_raidz_io_start_read()
1745 raidz_row_t *rr = rm->rm_row[0]; in vdev_raidz_io_start() local
1748 vdev_raidz_io_start_write(zio, rr, tvd->vdev_ashift); in vdev_raidz_io_start()
1751 vdev_raidz_io_start_read(zio, rr); in vdev_raidz_io_start()
1806 raidz_parity_verify(zio_t *zio, raidz_row_t *rr) in raidz_parity_verify() argument
1820 for (c = 0; c < rr->rr_firstdatacol; c++) { in raidz_parity_verify()
1821 rc = &rr->rr_col[c]; in raidz_parity_verify()
1834 if (rr->rr_nempty && rr->rr_abd_empty != NULL) in raidz_parity_verify()
1835 ret += vdev_draid_map_verify_empty(zio, rr); in raidz_parity_verify()
1842 vdev_raidz_generate_parity_row(rm, rr); in raidz_parity_verify()
1844 for (c = 0; c < rr->rr_firstdatacol; c++) { in raidz_parity_verify()
1845 rc = &rr->rr_col[c]; in raidz_parity_verify()
1862 vdev_raidz_worst_error(raidz_row_t *rr) in vdev_raidz_worst_error() argument
1866 for (int c = 0; c < rr->rr_cols; c++) in vdev_raidz_worst_error()
1867 error = zio_worst_error(error, rr->rr_col[c].rc_error); in vdev_raidz_worst_error()
1873 vdev_raidz_io_done_verified(zio_t *zio, raidz_row_t *rr) in vdev_raidz_io_done_verified() argument
1882 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_verified()
1883 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_verified()
1886 if (c < rr->rr_firstdatacol) in vdev_raidz_io_done_verified()
1893 } else if (c < rr->rr_firstdatacol && !rc->rc_tried) { in vdev_raidz_io_done_verified()
1910 rr->rr_firstdatacol - data_errors || in vdev_raidz_io_done_verified()
1912 int n = raidz_parity_verify(zio, rr); in vdev_raidz_io_done_verified()
1921 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_verified()
1922 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_verified()
1948 raidz_row_t *rr = rm->rm_row[i]; in raidz_restore_orig_data() local
1949 for (int c = 0; c < rr->rr_cols; c++) { in raidz_restore_orig_data()
1950 raidz_col_t *rc = &rr->rr_col[c]; in raidz_restore_orig_data()
1972 raidz_row_t *rr = rm->rm_row[r]; in raidz_reconstruct() local
1978 for (int c = 0; c < rr->rr_cols; c++) { in raidz_reconstruct()
1979 raidz_col_t *rc = &rr->rr_col[c]; in raidz_reconstruct()
2014 vdev_raidz_reconstruct_row(rm, rr, my_tgts, t); in raidz_reconstruct()
2022 raidz_row_t *rr = rm->rm_row[i]; in raidz_reconstruct() local
2024 for (int c = 0; c < rr->rr_cols; c++) { in raidz_reconstruct()
2025 raidz_col_t *rc = &rr->rr_col[c]; in raidz_reconstruct()
2038 c >= rr->rr_firstdatacol) { in raidz_reconstruct()
2048 vdev_raidz_io_done_verified(zio, rr); in raidz_reconstruct()
2116 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_combrec() local
2119 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_combrec()
2120 if (rr->rr_col[c].rc_error) in vdev_raidz_combrec()
2125 return (vdev_raidz_worst_error(rr)); in vdev_raidz_combrec()
2198 raidz_row_t *rr = rm->rm_row[row]; in vdev_raidz_reconstruct() local
2199 vdev_raidz_reconstruct_row(rm, rr, t, nt); in vdev_raidz_reconstruct()
2213 vdev_raidz_io_done_write_impl(zio_t *zio, raidz_row_t *rr) in vdev_raidz_io_done_write_impl() argument
2217 ASSERT3U(rr->rr_missingparity, <=, rr->rr_firstdatacol); in vdev_raidz_io_done_write_impl()
2218 ASSERT3U(rr->rr_missingdata, <=, rr->rr_cols - rr->rr_firstdatacol); in vdev_raidz_io_done_write_impl()
2221 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_write_impl()
2222 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_write_impl()
2241 if (total_errors > rr->rr_firstdatacol) { in vdev_raidz_io_done_write_impl()
2243 vdev_raidz_worst_error(rr)); in vdev_raidz_io_done_write_impl()
2249 raidz_row_t *rr) in vdev_raidz_io_done_reconstruct_known_missing() argument
2256 ASSERT3U(rr->rr_missingparity, <=, rr->rr_firstdatacol); in vdev_raidz_io_done_reconstruct_known_missing()
2257 ASSERT3U(rr->rr_missingdata, <=, rr->rr_cols - rr->rr_firstdatacol); in vdev_raidz_io_done_reconstruct_known_missing()
2260 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_reconstruct_known_missing()
2261 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_reconstruct_known_missing()
2277 if (c < rr->rr_firstdatacol) in vdev_raidz_io_done_reconstruct_known_missing()
2283 } else if (c < rr->rr_firstdatacol && !rc->rc_tried) { in vdev_raidz_io_done_reconstruct_known_missing()
2294 total_errors <= rr->rr_firstdatacol - parity_untried) { in vdev_raidz_io_done_reconstruct_known_missing()
2303 ASSERT(parity_errors < rr->rr_firstdatacol); in vdev_raidz_io_done_reconstruct_known_missing()
2310 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_io_done_reconstruct_known_missing()
2311 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_reconstruct_known_missing()
2318 ASSERT(rr->rr_firstdatacol >= n); in vdev_raidz_io_done_reconstruct_known_missing()
2320 vdev_raidz_reconstruct_row(rm, rr, tgts, n); in vdev_raidz_io_done_reconstruct_known_missing()
2328 vdev_raidz_read_all(zio_t *zio, raidz_row_t *rr) in vdev_raidz_read_all() argument
2333 rr->rr_missingdata = 0; in vdev_raidz_read_all()
2334 rr->rr_missingparity = 0; in vdev_raidz_read_all()
2341 if (rr->rr_nempty && rr->rr_abd_empty == NULL) in vdev_raidz_read_all()
2342 vdev_draid_map_alloc_empty(zio, rr); in vdev_raidz_read_all()
2344 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_read_all()
2345 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_read_all()
2371 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_io_done_unrecoverable() local
2373 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_unrecoverable()
2374 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_unrecoverable()
2405 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_io_done() local
2407 rm, rr); in vdev_raidz_io_done()
2412 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_io_done() local
2413 vdev_raidz_io_done_verified(zio, rr); in vdev_raidz_io_done()