1 /* $NetBSD: rf_disks.c,v 1.95 2023/09/25 16:16:50 oster Exp $ */
2 /*-
3 * Copyright (c) 1999 The NetBSD Foundation, Inc.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to The NetBSD Foundation
7 * by Greg Oster
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 /*
32 * Copyright (c) 1995 Carnegie-Mellon University.
33 * All rights reserved.
34 *
35 * Author: Mark Holland
36 *
37 * Permission to use, copy, modify and distribute this software and
38 * its documentation is hereby granted, provided that both the copyright
39 * notice and this permission notice appear in all copies of the
40 * software, derivative works or modified versions, and any portions
41 * thereof, and that both notices appear in supporting documentation.
42 *
43 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
44 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
45 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
46 *
47 * Carnegie Mellon requests users of this software to return to
48 *
49 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
50 * School of Computer Science
51 * Carnegie Mellon University
52 * Pittsburgh PA 15213-3890
53 *
54 * any improvements or extensions that they make and grant Carnegie the
55 * rights to redistribute these changes.
56 */
57
58 /***************************************************************
59 * rf_disks.c -- code to perform operations on the actual disks
60 ***************************************************************/
61
62 #include <sys/cdefs.h>
63 __KERNEL_RCSID(0, "$NetBSD: rf_disks.c,v 1.95 2023/09/25 16:16:50 oster Exp $");
64
65 #include <dev/raidframe/raidframevar.h>
66
67 #include "rf_raid.h"
68 #include "rf_alloclist.h"
69 #include "rf_driver.h"
70 #include "rf_utils.h"
71 #include "rf_general.h"
72 #include "rf_options.h"
73 #include "rf_kintf.h"
74 #include "rf_netbsd.h"
75
76 #include <sys/param.h>
77 #include <sys/systm.h>
78 #include <sys/proc.h>
79 #include <sys/ioctl.h>
80 #include <sys/fcntl.h>
81 #include <sys/vnode.h>
82 #include <sys/namei.h> /* for pathbuf */
83 #include <sys/kauth.h>
84 #include <miscfs/specfs/specdev.h> /* for v_rdev */
85
86 static int rf_AllocDiskStructures(RF_Raid_t *, RF_Config_t *);
87 static void rf_print_label_status( RF_Raid_t *, int, char *,
88 RF_ComponentLabel_t *);
89 static int rf_check_label_vitals( RF_Raid_t *, int, int, char *,
90 RF_ComponentLabel_t *, int, int );
91
92 #define DPRINTF6(a,b,c,d,e,f) if (rf_diskDebug) printf(a,b,c,d,e,f)
93 #define DPRINTF7(a,b,c,d,e,f,g) if (rf_diskDebug) printf(a,b,c,d,e,f,g)
94
95 /**************************************************************************
96 *
97 * initialize the disks comprising the array
98 *
99 * We want the spare disks to have regular row,col numbers so that we can
100 * easily substitue a spare for a failed disk. But, the driver code assumes
101 * throughout that the array contains numRow by numCol _non-spare_ disks, so
102 * it's not clear how to fit in the spares. This is an unfortunate holdover
103 * from raidSim. The quick and dirty fix is to make row zero bigger than the
104 * rest, and put all the spares in it. This probably needs to get changed
105 * eventually.
106 *
107 **************************************************************************/
108
109 int
rf_ConfigureDisks(RF_ShutdownList_t ** listp,RF_Raid_t * raidPtr,RF_Config_t * cfgPtr)110 rf_ConfigureDisks(RF_ShutdownList_t **listp, RF_Raid_t *raidPtr,
111 RF_Config_t *cfgPtr)
112 {
113 RF_RaidDisk_t *disks;
114 RF_SectorCount_t min_numblks = (RF_SectorCount_t) 0x7FFFFFFFFFFFLL;
115 RF_RowCol_t c;
116 int bs, ret;
117 unsigned i, count, foundone = 0, numFailuresThisRow;
118 int force;
119
120 force = cfgPtr->force;
121
122 ret = rf_AllocDiskStructures(raidPtr, cfgPtr);
123 if (ret)
124 goto fail;
125
126 disks = raidPtr->Disks;
127
128 numFailuresThisRow = 0;
129 for (c = 0; c < raidPtr->numCol; c++) {
130 ret = rf_ConfigureDisk(raidPtr,
131 &cfgPtr->devnames[0][c][0],
132 &disks[c], c);
133
134 if (ret)
135 goto fail;
136
137 if (disks[c].status == rf_ds_optimal) {
138 ret = raidfetch_component_label(raidPtr, c);
139 if (ret)
140 goto fail;
141
142 /* mark it as failed if the label looks bogus... */
143 if (!rf_reasonable_label(&raidPtr->raid_cinfo[c].ci_label,0) && !force) {
144 disks[c].status = rf_ds_failed;
145 }
146 }
147
148 if (disks[c].status != rf_ds_optimal) {
149 numFailuresThisRow++;
150 } else {
151 if (disks[c].numBlocks < min_numblks)
152 min_numblks = disks[c].numBlocks;
153 DPRINTF6("Disk at col %d: dev %s numBlocks %" PRIu64 " blockSize %d (%ld MB)\n",
154 c, disks[c].devname,
155 disks[c].numBlocks,
156 disks[c].blockSize,
157 (long int) disks[c].numBlocks *
158 disks[c].blockSize / 1024 / 1024);
159 }
160 }
161 /* XXX fix for n-fault tolerant */
162 /* XXX this should probably check to see how many failures
163 we can handle for this configuration! */
164 if (numFailuresThisRow > 0)
165 raidPtr->status = rf_rs_degraded;
166
167 /* all disks must be the same size & have the same block size, bs must
168 * be a power of 2 */
169 bs = 0;
170 foundone = 0;
171 for (c = 0; c < raidPtr->numCol; c++) {
172 if (disks[c].status == rf_ds_optimal) {
173 bs = disks[c].blockSize;
174 foundone = 1;
175 break;
176 }
177 }
178 if (!foundone) {
179 RF_ERRORMSG("RAIDFRAME: Did not find any live disks in the array.\n");
180 ret = EINVAL;
181 goto fail;
182 }
183 for (count = 0, i = 1; i; i <<= 1)
184 if (bs & i)
185 count++;
186 if (count != 1) {
187 RF_ERRORMSG1("Error: block size on disks (%d) must be a power of 2\n", bs);
188 ret = EINVAL;
189 goto fail;
190 }
191
192 if (rf_CheckLabels( raidPtr, cfgPtr )) {
193 printf("raid%d: There were fatal errors\n", raidPtr->raidid);
194 if (force != 0) {
195 printf("raid%d: Fatal errors being ignored.\n",
196 raidPtr->raidid);
197 } else {
198 ret = EINVAL;
199 goto fail;
200 }
201 }
202
203 for (c = 0; c < raidPtr->numCol; c++) {
204 if (disks[c].status == rf_ds_optimal) {
205 if (disks[c].blockSize != bs) {
206 RF_ERRORMSG1("Error: block size of disk at c %d different from disk at c 0\n", c);
207 ret = EINVAL;
208 goto fail;
209 }
210 if (disks[c].numBlocks != min_numblks) {
211 RF_ERRORMSG2("WARNING: truncating disk at c %d to %d blocks\n",
212 c, (int) min_numblks);
213 disks[c].numBlocks = min_numblks;
214 }
215 }
216 }
217
218 raidPtr->sectorsPerDisk = min_numblks;
219 raidPtr->logBytesPerSector = ffs(bs) - 1;
220 raidPtr->bytesPerSector = bs;
221 raidPtr->sectorMask = bs - 1;
222 return (0);
223
224 fail:
225
226 rf_UnconfigureVnodes( raidPtr );
227
228 return (ret);
229 }
230
231
232 /****************************************************************************
233 * set up the data structures describing the spare disks in the array
234 * recall from the above comment that the spare disk descriptors are stored
235 * in row zero, which is specially expanded to hold them.
236 ****************************************************************************/
237 int
rf_ConfigureSpareDisks(RF_ShutdownList_t ** listp,RF_Raid_t * raidPtr,RF_Config_t * cfgPtr)238 rf_ConfigureSpareDisks(RF_ShutdownList_t **listp, RF_Raid_t *raidPtr,
239 RF_Config_t *cfgPtr)
240 {
241 int i, ret;
242 unsigned int bs;
243 RF_RaidDisk_t *disks;
244 int num_spares_done;
245
246 num_spares_done = 0;
247
248 /* The space for the spares should have already been allocated by
249 * ConfigureDisks() */
250
251 disks = &raidPtr->Disks[raidPtr->numCol];
252 for (i = 0; i < raidPtr->numSpare; i++) {
253 ret = rf_ConfigureDisk(raidPtr, &cfgPtr->spare_names[i][0],
254 &disks[i], raidPtr->numCol + i);
255 if (ret)
256 goto fail;
257 if (disks[i].status != rf_ds_optimal) {
258 RF_ERRORMSG1("Warning: spare disk %s failed TUR\n",
259 &cfgPtr->spare_names[i][0]);
260 } else {
261 disks[i].status = rf_ds_spare; /* change status to
262 * spare */
263 DPRINTF6("Spare Disk %d: dev %s numBlocks %" PRIu64 " blockSize %d (%ld MB)\n", i,
264 disks[i].devname,
265 disks[i].numBlocks, disks[i].blockSize,
266 (long int) disks[i].numBlocks *
267 disks[i].blockSize / 1024 / 1024);
268 }
269 num_spares_done++;
270 }
271
272 /* check sizes and block sizes on spare disks */
273 bs = 1 << raidPtr->logBytesPerSector;
274 for (i = 0; i < raidPtr->numSpare; i++) {
275 if (disks[i].blockSize != bs) {
276 RF_ERRORMSG3("Block size of %d on spare disk %s is not the same as on other disks (%d)\n", disks[i].blockSize, disks[i].devname, bs);
277 ret = EINVAL;
278 goto fail;
279 }
280 if (disks[i].numBlocks < raidPtr->sectorsPerDisk) {
281 RF_ERRORMSG3("Spare disk %s (%d blocks) is too small to serve as a spare (need %" PRIu64 " blocks)\n",
282 disks[i].devname, disks[i].blockSize,
283 raidPtr->sectorsPerDisk);
284 ret = EINVAL;
285 goto fail;
286 } else
287 if (disks[i].numBlocks > raidPtr->sectorsPerDisk) {
288 RF_ERRORMSG3("Warning: truncating spare disk %s to %" PRIu64 " blocks (from %" PRIu64 ")\n",
289 disks[i].devname,
290 raidPtr->sectorsPerDisk,
291 disks[i].numBlocks);
292
293 disks[i].numBlocks = raidPtr->sectorsPerDisk;
294 }
295 }
296
297 return (0);
298
299 fail:
300
301 /* Release the hold on the main components. We've failed to allocate
302 * a spare, and since we're failing, we need to free things..
303
304 XXX failing to allocate a spare is *not* that big of a deal...
305 We *can* survive without it, if need be, esp. if we get hot
306 adding working.
307
308 If we don't fail out here, then we need a way to remove this spare...
309 that should be easier to do here than if we are "live"...
310
311 */
312
313 rf_UnconfigureVnodes( raidPtr );
314
315 return (ret);
316 }
317
318 static int
rf_AllocDiskStructures(RF_Raid_t * raidPtr,RF_Config_t * cfgPtr)319 rf_AllocDiskStructures(RF_Raid_t *raidPtr, RF_Config_t *cfgPtr)
320 {
321 int ret;
322 size_t entries = raidPtr->numCol + RF_MAXSPARE;
323
324 /* We allocate RF_MAXSPARE on the first row so that we
325 have room to do hot-swapping of spares */
326 raidPtr->Disks = RF_MallocAndAdd(
327 entries * sizeof(*raidPtr->Disks), raidPtr->cleanupList);
328 if (raidPtr->Disks == NULL) {
329 ret = ENOMEM;
330 goto fail;
331 }
332
333 /* get space for device specific stuff.. */
334 raidPtr->raid_cinfo = RF_MallocAndAdd(
335 entries * sizeof(*raidPtr->raid_cinfo), raidPtr->cleanupList);
336 if (raidPtr->raid_cinfo == NULL) {
337 ret = ENOMEM;
338 goto fail;
339 }
340
341 raidPtr->abortRecon = RF_MallocAndAdd(
342 entries * sizeof(int), raidPtr->cleanupList);
343 if (raidPtr->abortRecon == NULL) {
344 ret = ENOMEM;
345 goto fail;
346 }
347
348
349 return(0);
350 fail:
351 rf_UnconfigureVnodes( raidPtr );
352
353 return(ret);
354 }
355
356
357 /* configure a single disk during auto-configuration at boot */
358 int
rf_AutoConfigureDisks(RF_Raid_t * raidPtr,RF_Config_t * cfgPtr,RF_AutoConfig_t * auto_config)359 rf_AutoConfigureDisks(RF_Raid_t *raidPtr, RF_Config_t *cfgPtr,
360 RF_AutoConfig_t *auto_config)
361 {
362 RF_RaidDisk_t *disks;
363 RF_RaidDisk_t *diskPtr;
364 RF_RowCol_t c;
365 RF_SectorCount_t min_numblks = (RF_SectorCount_t) 0x7FFFFFFFFFFFLL;
366 int bs, ret;
367 int numFailuresThisRow;
368 RF_AutoConfig_t *ac;
369 int parity_good;
370 int mod_counter;
371 int mod_counter_found;
372
373 #if DEBUG
374 printf("Starting autoconfiguration of RAID set...\n");
375 #endif
376
377 ret = rf_AllocDiskStructures(raidPtr, cfgPtr);
378 if (ret)
379 goto fail;
380
381 disks = raidPtr->Disks;
382
383 /* assume the parity will be fine.. */
384 parity_good = RF_RAID_CLEAN;
385
386 /* Check for mod_counters that are too low */
387 mod_counter_found = 0;
388 mod_counter = 0;
389 ac = auto_config;
390 while(ac!=NULL) {
391 if (mod_counter_found==0) {
392 mod_counter = ac->clabel->mod_counter;
393 mod_counter_found = 1;
394 } else {
395 if (ac->clabel->mod_counter > mod_counter) {
396 mod_counter = ac->clabel->mod_counter;
397 }
398 }
399 ac->flag = 0; /* clear the general purpose flag */
400 ac = ac->next;
401 }
402
403 bs = 0;
404
405 numFailuresThisRow = 0;
406 for (c = 0; c < raidPtr->numCol; c++) {
407 diskPtr = &disks[c];
408
409 /* find this row/col in the autoconfig */
410 #if DEBUG
411 printf("Looking for %d in autoconfig\n",c);
412 #endif
413 ac = auto_config;
414 while(ac!=NULL) {
415 if (ac->clabel==NULL) {
416 /* big-time bad news. */
417 goto fail;
418 }
419 if ((ac->clabel->column == c) &&
420 (ac->clabel->mod_counter == mod_counter)) {
421 /* it's this one... */
422 /* flag it as 'used', so we don't
423 free it later. */
424 ac->flag = 1;
425 #if DEBUG
426 printf("Found: %s at %d\n",
427 ac->devname,c);
428 #endif
429
430 break;
431 }
432 ac=ac->next;
433 }
434
435 if (ac==NULL) {
436 /* we didn't find an exact match with a
437 correct mod_counter above... can we find
438 one with an incorrect mod_counter to use
439 instead? (this one, if we find it, will be
440 marked as failed once the set configures)
441 */
442
443 ac = auto_config;
444 while(ac!=NULL) {
445 if (ac->clabel==NULL) {
446 /* big-time bad news. */
447 goto fail;
448 }
449 if (ac->clabel->column == c) {
450 /* it's this one...
451 flag it as 'used', so we
452 don't free it later. */
453 ac->flag = 1;
454 #if DEBUG
455 printf("Found(low mod_counter): %s at %d\n",
456 ac->devname,c);
457 #endif
458
459 break;
460 }
461 ac=ac->next;
462 }
463 }
464
465
466
467 if (ac!=NULL) {
468 /* Found it. Configure it.. */
469 diskPtr->blockSize = ac->clabel->blockSize;
470 diskPtr->numBlocks =
471 rf_component_label_numblocks(ac->clabel);
472 /* Note: rf_protectedSectors is already
473 factored into numBlocks here */
474 raidPtr->raid_cinfo[c].ci_vp = ac->vp;
475 raidPtr->raid_cinfo[c].ci_dev = ac->dev;
476
477 memcpy(raidget_component_label(raidPtr, c),
478 ac->clabel, sizeof(*ac->clabel));
479 snprintf(diskPtr->devname, sizeof(diskPtr->devname),
480 "/dev/%s", ac->devname);
481
482 /* note the fact that this component was
483 autoconfigured. You'll need this info
484 later. Trust me :) */
485 diskPtr->auto_configured = 1;
486 diskPtr->dev = ac->dev;
487
488 /*
489 * we allow the user to specify that
490 * only a fraction of the disks should
491 * be used this is just for debug: it
492 * speeds up the parity scan
493 */
494
495 diskPtr->numBlocks = diskPtr->numBlocks *
496 rf_sizePercentage / 100;
497
498 /* XXX these will get set multiple times,
499 but since we're autoconfiguring, they'd
500 better be always the same each time!
501 If not, this is the least of your worries */
502
503 bs = diskPtr->blockSize;
504 min_numblks = diskPtr->numBlocks;
505
506 /* this gets done multiple times, but that's
507 fine -- the serial number will be the same
508 for all components, guaranteed */
509 raidPtr->serial_number = ac->clabel->serial_number;
510 /* check the last time the label was modified */
511
512 if (ac->clabel->mod_counter != mod_counter) {
513 /* Even though we've filled in all of
514 the above, we don't trust this
515 component since its modification
516 counter is not in sync with the
517 rest, and we really consider it to
518 be failed. */
519 disks[c].status = rf_ds_failed;
520 numFailuresThisRow++;
521 } else {
522 if (ac->clabel->clean != RF_RAID_CLEAN) {
523 parity_good = RF_RAID_DIRTY;
524 }
525 }
526 } else {
527 /* Didn't find it at all!! Component must
528 really be dead */
529 disks[c].status = rf_ds_failed;
530 snprintf(disks[c].devname, sizeof(disks[c].devname),
531 "component%d", c);
532 numFailuresThisRow++;
533 }
534 }
535 /* XXX fix for n-fault tolerant */
536 /* XXX this should probably check to see how many failures
537 we can handle for this configuration! */
538 if (numFailuresThisRow > 0) {
539 raidPtr->status = rf_rs_degraded;
540 raidPtr->numFailures = numFailuresThisRow;
541 }
542
543 /* close the device for the ones that didn't get used */
544
545 ac = auto_config;
546 while(ac!=NULL) {
547 if (ac->flag == 0) {
548 vn_lock(ac->vp, LK_EXCLUSIVE | LK_RETRY);
549 VOP_CLOSE(ac->vp, FREAD | FWRITE, NOCRED);
550 vput(ac->vp);
551 ac->vp = NULL;
552 #if DEBUG
553 printf("Released %s from auto-config set.\n",
554 ac->devname);
555 #endif
556 }
557 ac = ac->next;
558 }
559
560 raidPtr->mod_counter = mod_counter;
561
562 /* note the state of the parity, if any */
563 raidPtr->parity_good = parity_good;
564 raidPtr->sectorsPerDisk = min_numblks;
565 raidPtr->logBytesPerSector = ffs(bs) - 1;
566 raidPtr->bytesPerSector = bs;
567 raidPtr->sectorMask = bs - 1;
568 return (0);
569
570 fail:
571
572 rf_UnconfigureVnodes( raidPtr );
573
574 return (ret);
575
576 }
577
578 /* configure a single disk in the array */
579 int
rf_ConfigureDisk(RF_Raid_t * raidPtr,char * bf,RF_RaidDisk_t * diskPtr,RF_RowCol_t col)580 rf_ConfigureDisk(RF_Raid_t *raidPtr, char *bf, RF_RaidDisk_t *diskPtr,
581 RF_RowCol_t col)
582 {
583 char *p;
584 struct pathbuf *pb;
585 struct vnode *vp;
586 int error;
587
588 p = rf_find_non_white(bf);
589 if (p[strlen(p) - 1] == '\n') {
590 /* strip off the newline */
591 p[strlen(p) - 1] = '\0';
592 }
593 (void) strcpy(diskPtr->devname, p);
594
595 /* Let's start by claiming the component is fine and well... */
596 diskPtr->status = rf_ds_optimal;
597
598 raidPtr->raid_cinfo[col].ci_vp = NULL;
599 raidPtr->raid_cinfo[col].ci_dev = 0;
600
601 if (!strcmp("absent", diskPtr->devname)) {
602 printf("Ignoring missing component at column %d\n", col);
603 snprintf(diskPtr->devname, sizeof(diskPtr->devname),
604 "component%d", col);
605 diskPtr->status = rf_ds_failed;
606 return (0);
607 }
608
609 pb = pathbuf_create(diskPtr->devname);
610 if (pb == NULL) {
611 printf("pathbuf_create for device: %s failed!\n",
612 diskPtr->devname);
613 return ENOMEM;
614 }
615 error = vn_bdev_openpath(pb, &vp, curlwp);
616 pathbuf_destroy(pb);
617 if (error) {
618 printf("open device: '%s' failed: %d\n", diskPtr->devname, error);
619 if (error == ENXIO) {
620 /* the component isn't there... must be dead :-( */
621 diskPtr->status = rf_ds_failed;
622 return 0;
623 } else {
624 return (error);
625 }
626 }
627
628 if ((error = rf_getdisksize(vp, diskPtr)) != 0)
629 return (error);
630
631 /*
632 * If this raidPtr's bytesPerSector is zero, fill it in with this
633 * components blockSize. This will give us something to work with
634 * initially, and if it is wrong, we'll get errors later.
635 */
636 if (raidPtr->bytesPerSector == 0)
637 raidPtr->bytesPerSector = diskPtr->blockSize;
638
639 if (diskPtr->status == rf_ds_optimal) {
640 raidPtr->raid_cinfo[col].ci_vp = vp;
641 raidPtr->raid_cinfo[col].ci_dev = vp->v_rdev;
642
643 /* This component was not automatically configured */
644 diskPtr->auto_configured = 0;
645 diskPtr->dev = vp->v_rdev;
646
647 /* we allow the user to specify that only a fraction of the
648 * disks should be used this is just for debug: it speeds up
649 * the parity scan */
650 diskPtr->numBlocks = diskPtr->numBlocks *
651 rf_sizePercentage / 100;
652 }
653 return (0);
654 }
655
656 static void
rf_print_label_status(RF_Raid_t * raidPtr,int column,char * dev_name,RF_ComponentLabel_t * ci_label)657 rf_print_label_status(RF_Raid_t *raidPtr, int column, char *dev_name,
658 RF_ComponentLabel_t *ci_label)
659 {
660
661 printf("raid%d: Component %s being configured at col: %d\n",
662 raidPtr->raidid, dev_name, column );
663 printf(" Column: %d Num Columns: %d\n",
664 ci_label->column,
665 ci_label->num_columns);
666 printf(" Version: %d Serial Number: %d Mod Counter: %d\n",
667 ci_label->version, ci_label->serial_number,
668 ci_label->mod_counter);
669 printf(" Clean: %s Status: %d\n",
670 ci_label->clean ? "Yes" : "No", ci_label->status );
671 }
672
rf_check_label_vitals(RF_Raid_t * raidPtr,int row,int column,char * dev_name,RF_ComponentLabel_t * ci_label,int serial_number,int mod_counter)673 static int rf_check_label_vitals(RF_Raid_t *raidPtr, int row, int column,
674 char *dev_name, RF_ComponentLabel_t *ci_label,
675 int serial_number, int mod_counter)
676 {
677 int fatal_error = 0;
678
679 if (serial_number != ci_label->serial_number) {
680 printf("%s has a different serial number: %d %d\n",
681 dev_name, serial_number, ci_label->serial_number);
682 fatal_error = 1;
683 }
684 if (mod_counter != ci_label->mod_counter) {
685 printf("%s has a different modification count: %d %d\n",
686 dev_name, mod_counter, ci_label->mod_counter);
687 }
688
689 if (row != ci_label->row) {
690 printf("Row out of alignment for: %s\n", dev_name);
691 fatal_error = 1;
692 }
693 if (column != ci_label->column) {
694 printf("Column out of alignment for: %s\n", dev_name);
695 fatal_error = 1;
696 }
697 if (raidPtr->numCol != ci_label->num_columns) {
698 printf("Number of columns do not match for: %s\n", dev_name);
699 fatal_error = 1;
700 }
701 if (ci_label->clean == 0) {
702 /* it's not clean, but that's not fatal */
703 printf("%s is not clean!\n", dev_name);
704 }
705 return(fatal_error);
706 }
707
708
709 static void
rf_handle_hosed(RF_Raid_t * raidPtr,RF_Config_t * cfgPtr,int hosed_column,int again)710 rf_handle_hosed(RF_Raid_t *raidPtr, RF_Config_t *cfgPtr, int hosed_column,
711 int again)
712 {
713 printf("Hosed component: %s\n", &cfgPtr->devnames[0][hosed_column][0]);
714 if (cfgPtr->force)
715 return;
716
717 /* we'll fail this component, as if there are
718 other major errors, we aren't forcing things
719 and we'll abort the config anyways */
720 if (again && raidPtr->Disks[hosed_column].status == rf_ds_failed)
721 return;
722
723 raidPtr->Disks[hosed_column].status = rf_ds_failed;
724 raidPtr->numFailures++;
725 raidPtr->status = rf_rs_degraded;
726 }
727
728 /*
729
730 rf_CheckLabels() - check all the component labels for consistency.
731 Return an error if there is anything major amiss.
732
733 */
734
735 int
rf_CheckLabels(RF_Raid_t * raidPtr,RF_Config_t * cfgPtr)736 rf_CheckLabels(RF_Raid_t *raidPtr, RF_Config_t *cfgPtr)
737 {
738 int c;
739 char *dev_name;
740 RF_ComponentLabel_t *ci_label;
741 int serial_number = 0;
742 int mod_number = 0;
743 int fatal_error = 0;
744 int mod_values[4];
745 int mod_count[4];
746 int ser_values[4];
747 int ser_count[4];
748 int num_ser;
749 int num_mod;
750 int i;
751 int found;
752 int hosed_column;
753 int too_fatal;
754 int parity_good;
755
756 hosed_column = -1;
757 too_fatal = 0;
758
759 /*
760 We're going to try to be a little intelligent here. If one
761 component's label is bogus, and we can identify that it's the
762 *only* one that's gone, we'll mark it as "failed" and allow
763 the configuration to proceed. This will be the *only* case
764 that we'll proceed if there would be (otherwise) fatal errors.
765
766 Basically we simply keep a count of how many components had
767 what serial number. If all but one agree, we simply mark
768 the disagreeing component as being failed, and allow
769 things to come up "normally".
770
771 We do this first for serial numbers, and then for "mod_counter".
772
773 */
774
775 num_ser = 0;
776 num_mod = 0;
777
778 ser_values[0] = ser_values[1] = ser_values[2] = ser_values[3] = 0;
779 ser_count[0] = ser_count[1] = ser_count[2] = ser_count[3] = 0;
780 mod_values[0] = mod_values[1] = mod_values[2] = mod_values[3] = 0;
781 mod_count[0] = mod_count[1] = mod_count[2] = mod_count[3] = 0;
782
783 for (c = 0; c < raidPtr->numCol; c++) {
784 if (raidPtr->Disks[c].status != rf_ds_optimal)
785 continue;
786 ci_label = raidget_component_label(raidPtr, c);
787 found=0;
788 for(i=0;i<num_ser;i++) {
789 if (ser_values[i] == ci_label->serial_number) {
790 ser_count[i]++;
791 found=1;
792 break;
793 }
794 }
795 if (!found) {
796 ser_values[num_ser] = ci_label->serial_number;
797 ser_count[num_ser] = 1;
798 num_ser++;
799 if (num_ser>2) {
800 fatal_error = 1;
801 break;
802 }
803 }
804 found=0;
805 for(i=0;i<num_mod;i++) {
806 if (mod_values[i] == ci_label->mod_counter) {
807 mod_count[i]++;
808 found=1;
809 break;
810 }
811 }
812 if (!found) {
813 mod_values[num_mod] = ci_label->mod_counter;
814 mod_count[num_mod] = 1;
815 num_mod++;
816 if (num_mod>2) {
817 fatal_error = 1;
818 break;
819 }
820 }
821 }
822 #if DEBUG
823 printf("raid%d: Summary of serial numbers:\n", raidPtr->raidid);
824 for(i=0;i<num_ser;i++) {
825 printf("%d %d\n", ser_values[i], ser_count[i]);
826 }
827 printf("raid%d: Summary of mod counters:\n", raidPtr->raidid);
828 for(i=0;i<num_mod;i++) {
829 printf("%d %d\n", mod_values[i], mod_count[i]);
830 }
831 #endif
832 serial_number = ser_values[0];
833 if (num_ser == 2) {
834 if ((ser_count[0] == 1) || (ser_count[1] == 1)) {
835 /* Locate the maverick component */
836 if (ser_count[1] > ser_count[0]) {
837 serial_number = ser_values[1];
838 }
839
840 for (c = 0; c < raidPtr->numCol; c++) {
841 if (raidPtr->Disks[c].status != rf_ds_optimal)
842 continue;
843 ci_label = raidget_component_label(raidPtr, c);
844 if (serial_number != ci_label->serial_number) {
845 hosed_column = c;
846 break;
847 }
848 }
849 if (hosed_column != -1)
850 rf_handle_hosed(raidPtr, cfgPtr, hosed_column,
851 0);
852 } else {
853 too_fatal = 1;
854 }
855 if (cfgPtr->parityConfig == '0') {
856 /* We've identified two different serial numbers.
857 RAID 0 can't cope with that, so we'll punt */
858 too_fatal = 1;
859 }
860
861 }
862
863 /* record the serial number for later. If we bail later, setting
864 this doesn't matter, otherwise we've got the best guess at the
865 correct serial number */
866 raidPtr->serial_number = serial_number;
867
868 mod_number = mod_values[0];
869 if (num_mod == 2) {
870 if ((mod_count[0] == 1) || (mod_count[1] == 1)) {
871 /* Locate the maverick component */
872 if (mod_count[1] > mod_count[0]) {
873 mod_number = mod_values[1];
874 } else if (mod_count[1] < mod_count[0]) {
875 mod_number = mod_values[0];
876 } else {
877 /* counts of different modification values
878 are the same. Assume greater value is
879 the correct one, all other things
880 considered */
881 if (mod_values[0] > mod_values[1]) {
882 mod_number = mod_values[0];
883 } else {
884 mod_number = mod_values[1];
885 }
886
887 }
888
889 for (c = 0; c < raidPtr->numCol; c++) {
890 if (raidPtr->Disks[c].status != rf_ds_optimal)
891 continue;
892
893 ci_label = raidget_component_label(raidPtr, c);
894 if (mod_number != ci_label->mod_counter) {
895 if (hosed_column == c) {
896 /* same one. Can
897 deal with it. */
898 } else {
899 hosed_column = c;
900 if (num_ser != 1) {
901 too_fatal = 1;
902 break;
903 }
904 }
905 }
906 }
907 if (hosed_column != -1)
908 rf_handle_hosed(raidPtr, cfgPtr, hosed_column,
909 1);
910 } else {
911 too_fatal = 1;
912 }
913 if (cfgPtr->parityConfig == '0') {
914 /* We've identified two different mod counters.
915 RAID 0 can't cope with that, so we'll punt */
916 too_fatal = 1;
917 }
918 }
919
920 raidPtr->mod_counter = mod_number;
921
922 if (too_fatal) {
923 /* we've had both a serial number mismatch, and a mod_counter
924 mismatch -- and they involved two different components!!
925 Bail -- make things fail so that the user must force
926 the issue... */
927 hosed_column = -1;
928 fatal_error = 1;
929 }
930
931 if (num_ser > 2) {
932 printf("raid%d: Too many different serial numbers!\n",
933 raidPtr->raidid);
934 fatal_error = 1;
935 }
936
937 if (num_mod > 2) {
938 printf("raid%d: Too many different mod counters!\n",
939 raidPtr->raidid);
940 fatal_error = 1;
941 }
942
943 for (c = 0; c < raidPtr->numCol; c++) {
944 if (raidPtr->Disks[c].status != rf_ds_optimal) {
945 hosed_column = c;
946 break;
947 }
948 }
949
950 /* we start by assuming the parity will be good, and flee from
951 that notion at the slightest sign of trouble */
952
953 parity_good = RF_RAID_CLEAN;
954
955 for (c = 0; c < raidPtr->numCol; c++) {
956 dev_name = &cfgPtr->devnames[0][c][0];
957 ci_label = raidget_component_label(raidPtr, c);
958
959 if (c == hosed_column) {
960 printf("raid%d: Ignoring %s\n",
961 raidPtr->raidid, dev_name);
962 } else {
963 rf_print_label_status( raidPtr, c, dev_name, ci_label);
964 if (rf_check_label_vitals( raidPtr, 0, c,
965 dev_name, ci_label,
966 serial_number,
967 mod_number )) {
968 fatal_error = 1;
969 }
970 if (ci_label->clean != RF_RAID_CLEAN) {
971 parity_good = RF_RAID_DIRTY;
972 }
973 }
974 }
975
976 if (fatal_error) {
977 parity_good = RF_RAID_DIRTY;
978 }
979
980 /* we note the state of the parity */
981 raidPtr->parity_good = parity_good;
982
983 return(fatal_error);
984 }
985
986 int
rf_add_hot_spare(RF_Raid_t * raidPtr,RF_SingleComponent_t * sparePtr)987 rf_add_hot_spare(RF_Raid_t *raidPtr, RF_SingleComponent_t *sparePtr)
988 {
989 RF_DiskQueue_t *spareQueues;
990 RF_RaidDisk_t *disks;
991 int ret;
992 unsigned int bs;
993 int spare_number;
994
995 ret=0;
996
997 if (raidPtr->numSpare >= RF_MAXSPARE) {
998 RF_ERRORMSG1("Too many spares: %d\n", raidPtr->numSpare);
999 return(EINVAL);
1000 }
1001
1002 rf_lock_mutex2(raidPtr->mutex);
1003 while (raidPtr->changing_components == 1) {
1004 rf_wait_cond2(raidPtr->changing_components_cv, raidPtr->mutex);
1005 }
1006 raidPtr->changing_components = 1;
1007 rf_unlock_mutex2(raidPtr->mutex);
1008
1009 /* the beginning of the spares... */
1010 disks = &raidPtr->Disks[raidPtr->numCol];
1011
1012 spare_number = raidPtr->numSpare;
1013
1014 ret = rf_ConfigureDisk(raidPtr, sparePtr->component_name,
1015 &disks[spare_number],
1016 raidPtr->numCol + spare_number);
1017
1018 if (ret)
1019 goto fail;
1020 if (disks[spare_number].status != rf_ds_optimal) {
1021 RF_ERRORMSG1("Warning: spare disk %s failed TUR\n",
1022 sparePtr->component_name);
1023 rf_close_component(raidPtr, raidPtr->raid_cinfo[raidPtr->numCol+spare_number].ci_vp, 0);
1024 ret=EINVAL;
1025 goto fail;
1026 } else {
1027 disks[spare_number].status = rf_ds_spare;
1028 DPRINTF6("Spare Disk %d: dev %s numBlocks %" PRIu64 " blockSize %d (%ld MB)\n",
1029 spare_number,
1030 disks[spare_number].devname,
1031 disks[spare_number].numBlocks,
1032 disks[spare_number].blockSize,
1033 (long int) disks[spare_number].numBlocks *
1034 disks[spare_number].blockSize / 1024 / 1024);
1035 }
1036
1037
1038 /* check sizes and block sizes on the spare disk */
1039 bs = 1 << raidPtr->logBytesPerSector;
1040 if (disks[spare_number].blockSize != bs) {
1041 RF_ERRORMSG3("Block size of %d on spare disk %s is not the same as on other disks (%d)\n", disks[spare_number].blockSize, disks[spare_number].devname, bs);
1042 rf_close_component(raidPtr, raidPtr->raid_cinfo[raidPtr->numCol+spare_number].ci_vp, 0);
1043 ret = EINVAL;
1044 goto fail;
1045 }
1046 if (disks[spare_number].numBlocks < raidPtr->sectorsPerDisk) {
1047 RF_ERRORMSG3("Spare disk %s (%d blocks) is too small to serve as a spare (need %" PRIu64 " blocks)\n",
1048 disks[spare_number].devname,
1049 disks[spare_number].blockSize,
1050 raidPtr->sectorsPerDisk);
1051 rf_close_component(raidPtr, raidPtr->raid_cinfo[raidPtr->numCol+spare_number].ci_vp, 0);
1052 ret = EINVAL;
1053 goto fail;
1054 } else {
1055 if (disks[spare_number].numBlocks >
1056 raidPtr->sectorsPerDisk) {
1057 RF_ERRORMSG3("Warning: truncating spare disk %s to %" PRIu64 " blocks (from %" PRIu64 ")\n",
1058 disks[spare_number].devname,
1059 raidPtr->sectorsPerDisk,
1060 disks[spare_number].numBlocks);
1061
1062 disks[spare_number].numBlocks = raidPtr->sectorsPerDisk;
1063 }
1064 }
1065
1066 /*
1067 * We only grow one initialized diskQueue at a time
1068 * spare_number can be lower than raidPtr->maxQueue (update)
1069 * or they can be equal (initialize new queue)
1070 */
1071 RF_ASSERT(spare_number <= raidPtr->maxQueue);
1072
1073 spareQueues = &raidPtr->Queues[raidPtr->numCol];
1074 if (spare_number == raidPtr->maxQueue) {
1075 ret = rf_ConfigureDiskQueue(raidPtr, &spareQueues[spare_number],
1076 raidPtr->numCol + spare_number,
1077 raidPtr->qType,
1078 raidPtr->sectorsPerDisk,
1079 raidPtr->Disks[raidPtr->numCol +
1080 spare_number].dev,
1081 raidPtr->maxOutstanding,
1082 &raidPtr->shutdownList,
1083 raidPtr->cleanupList);
1084 if (ret)
1085 goto fail;
1086 rf_lock_mutex2(raidPtr->mutex);
1087 raidPtr->maxQueue++;
1088 rf_unlock_mutex2(raidPtr->mutex);
1089 } else {
1090 (void)rf_UpdateDiskQueue(&spareQueues[spare_number],
1091 &disks[spare_number]);
1092 }
1093
1094 fail:
1095 rf_lock_mutex2(raidPtr->mutex);
1096
1097 if (ret == 0) {
1098 raidPtr->numSpare++;
1099 }
1100
1101 raidPtr->changing_components = 0;
1102 rf_signal_cond2(raidPtr->changing_components_cv);
1103 rf_unlock_mutex2(raidPtr->mutex);
1104
1105 return(ret);
1106 }
1107
1108 int
rf_remove_hot_spare(RF_Raid_t * raidPtr,RF_SingleComponent_t * sparePtr)1109 rf_remove_hot_spare(RF_Raid_t *raidPtr, RF_SingleComponent_t *sparePtr)
1110 {
1111 int spare_number;
1112 int i;
1113 RF_RaidDisk_t *disk;
1114 struct vnode *vp;
1115 int ret = EINVAL;
1116
1117 spare_number = sparePtr->column - raidPtr->numCol;
1118 if (spare_number < 0 || spare_number > raidPtr->numSpare)
1119 return(ret);
1120
1121 rf_lock_mutex2(raidPtr->mutex);
1122 while (raidPtr->changing_components == 1) {
1123 rf_wait_cond2(raidPtr->changing_components_cv, raidPtr->mutex);
1124 }
1125 raidPtr->changing_components = 1;
1126 rf_unlock_mutex2(raidPtr->mutex);
1127
1128 rf_SuspendNewRequestsAndWait(raidPtr);
1129
1130 disk = &raidPtr->Disks[raidPtr->numCol + spare_number];
1131 if (disk->status != rf_ds_spare &&
1132 disk->status != rf_ds_failed) {
1133 printf("Spare is in use %d\n", disk->status);
1134 ret = EBUSY;
1135 goto out;
1136 }
1137
1138 vp = raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp;
1139 raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp = NULL;
1140 raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_dev = 0;
1141
1142 /* This component was not automatically configured */
1143 disk->auto_configured = 0;
1144 disk->dev = 0;
1145 disk->numBlocks = 0;
1146 disk->status = rf_ds_failed;
1147 snprintf(disk->devname, sizeof(disk->devname),
1148 "absent_spare%d", spare_number);
1149 rf_close_component(raidPtr, vp, 0);
1150
1151 rf_lock_mutex2(raidPtr->mutex);
1152
1153 /* at this point we know spare_number is to be pushed all the way to the end of the array... */
1154
1155 for (i = raidPtr->numCol + spare_number; i < raidPtr->numCol+raidPtr->numSpare-1; i++) {
1156 /* now we work our way up the spare array, swaping the current one for the next one */
1157 rf_swap_components(raidPtr, i, i+1);
1158 }
1159
1160 raidPtr->numSpare--;
1161 rf_unlock_mutex2(raidPtr->mutex);
1162
1163 rf_ResumeNewRequests(raidPtr);
1164
1165 ret = 0;
1166
1167 out:
1168
1169 rf_lock_mutex2(raidPtr->mutex);
1170 raidPtr->changing_components = 0;
1171 rf_signal_cond2(raidPtr->changing_components_cv);
1172 rf_unlock_mutex2(raidPtr->mutex);
1173
1174 return(ret);
1175 }
1176
1177 /*
1178 * Delete a non hot spare component
1179 */
1180 int
rf_delete_component(RF_Raid_t * raidPtr,RF_SingleComponent_t * component)1181 rf_delete_component(RF_Raid_t *raidPtr, RF_SingleComponent_t *component)
1182 {
1183 RF_RaidDisk_t *disk;
1184 RF_RowCol_t col = component->column;
1185 struct vnode *vp;
1186 int ret = EINVAL;
1187
1188 if (col < 0 || col >= raidPtr->numCol)
1189 return(ret);
1190
1191 rf_lock_mutex2(raidPtr->mutex);
1192 while (raidPtr->changing_components == 1) {
1193 rf_wait_cond2(raidPtr->changing_components_cv, raidPtr->mutex);
1194 }
1195 raidPtr->changing_components = 1;
1196 rf_unlock_mutex2(raidPtr->mutex);
1197
1198 disk = &raidPtr->Disks[col];
1199
1200 /* 1. This component must be marked as failed or spared */
1201 switch (disk->status) {
1202 case rf_ds_failed:
1203 case rf_ds_dist_spared:
1204 case rf_ds_spared:
1205 break;
1206 default:
1207 ret = EBUSY;
1208 goto out;
1209 }
1210
1211 vp = raidPtr->raid_cinfo[col].ci_vp;
1212 raidPtr->raid_cinfo[col].ci_vp = NULL;
1213 raidPtr->raid_cinfo[col].ci_dev = 0;
1214
1215 /* This component was not automatically configured */
1216 disk->auto_configured = 0;
1217 disk->dev = 0;
1218 disk->numBlocks = 0;
1219 snprintf(disk->devname, sizeof(disk->devname), "component%d", col);
1220
1221 rf_close_component(raidPtr, vp, 0);
1222
1223 ret = 0;
1224 out:
1225 rf_lock_mutex2(raidPtr->mutex);
1226 raidPtr->changing_components = 0;
1227 rf_signal_cond2(raidPtr->changing_components_cv);
1228 rf_unlock_mutex2(raidPtr->mutex);
1229
1230 return(ret);
1231 }
1232
1233 int
rf_remove_component(RF_Raid_t * raidPtr,RF_SingleComponent_t * component)1234 rf_remove_component(RF_Raid_t *raidPtr, RF_SingleComponent_t *component)
1235 {
1236 RF_RowCol_t col = component->column;
1237
1238 if (col < 0 || col >= raidPtr->numCol + raidPtr->numSpare)
1239 return(EINVAL);
1240
1241 if (col >= raidPtr->numCol)
1242 return rf_remove_hot_spare(raidPtr, component);
1243 else
1244 return rf_delete_component(raidPtr, component);
1245 }
1246
1247 int
rf_incorporate_hot_spare(RF_Raid_t * raidPtr,RF_SingleComponent_t * component)1248 rf_incorporate_hot_spare(RF_Raid_t *raidPtr,
1249 RF_SingleComponent_t *component)
1250 {
1251
1252 /* Issues here include how to 'move' this in if there is IO
1253 taking place (e.g. component queues and such) */
1254
1255 return(EINVAL); /* Not implemented yet. */
1256 }
1257
1258 void
rf_swap_components(RF_Raid_t * raidPtr,int a,int b)1259 rf_swap_components(RF_Raid_t *raidPtr, int a, int b)
1260 {
1261 char tmpdevname[56]; /* 56 is from raidframevar.h */
1262 RF_ComponentLabel_t tmp_ci_label;
1263 dev_t tmp_ci_dev, tmp_dev;
1264 int tmp_status;
1265 struct vnode *tmp_ci_vp;
1266
1267
1268 /* This function *MUST* be called with all IO suspended. */
1269 RF_ASSERT(raidPtr->accesses_suspended == 0);
1270
1271 /* Swap the component names... */
1272 snprintf(tmpdevname, sizeof(tmpdevname), "%s", raidPtr->Disks[a].devname);
1273 snprintf(raidPtr->Disks[a].devname, sizeof(raidPtr->Disks[a].devname), "%s", raidPtr->Disks[b].devname);
1274 snprintf(raidPtr->Disks[b].devname, sizeof(raidPtr->Disks[b].devname), "%s", tmpdevname);
1275
1276 /* and the vp */
1277 tmp_ci_vp = raidPtr->raid_cinfo[a].ci_vp;
1278 raidPtr->raid_cinfo[a].ci_vp = raidPtr->raid_cinfo[b].ci_vp;
1279 raidPtr->raid_cinfo[b].ci_vp = tmp_ci_vp;
1280
1281 /* and the ci dev */
1282 tmp_ci_dev = raidPtr->raid_cinfo[a].ci_dev;
1283 raidPtr->raid_cinfo[a].ci_dev = raidPtr->raid_cinfo[b].ci_dev;
1284 raidPtr->raid_cinfo[b].ci_dev = tmp_ci_dev;
1285
1286 /* the dev itself */
1287 tmp_dev = raidPtr->Disks[a].dev;
1288 raidPtr->Disks[a].dev = raidPtr->Disks[b].dev;
1289 raidPtr->Disks[b].dev = tmp_dev;
1290
1291 /* the component label */
1292 tmp_ci_label = raidPtr->raid_cinfo[a].ci_label;
1293 raidPtr->raid_cinfo[a].ci_label = raidPtr->raid_cinfo[b].ci_label;
1294 raidPtr->raid_cinfo[b].ci_label = tmp_ci_label;
1295
1296 /* and the status */
1297 tmp_status = raidPtr->Disks[a].status;
1298 raidPtr->Disks[a].status = raidPtr->Disks[b].status;
1299 raidPtr->Disks[b].status = tmp_status;
1300 }
1301
1302