1 /* $NetBSD: rf_raid.h,v 1.53 2023/09/25 21:59:38 oster Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /********************************************** 30 * rf_raid.h -- main header file for RAID driver 31 **********************************************/ 32 33 34 #ifndef _RF__RF_RAID_H_ 35 #define _RF__RF_RAID_H_ 36 37 #include <dev/raidframe/raidframevar.h> 38 #include "rf_archs.h" 39 #include "rf_threadstuff.h" 40 41 #include "rf_netbsd.h" 42 43 #include <sys/disklabel.h> 44 #include <sys/types.h> 45 #include <sys/queue.h> 46 47 #include "rf_alloclist.h" 48 #include "rf_stripelocks.h" 49 #include "rf_layout.h" 50 #include "rf_disks.h" 51 #include "rf_debugMem.h" 52 #include "rf_diskqueue.h" 53 #include "rf_reconstruct.h" 54 #include "rf_acctrace.h" 55 #include "rf_fifo.h" 56 57 #if RF_INCLUDE_PARITYLOGGING > 0 58 #include "rf_paritylog.h" 59 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */ 60 61 #define RF_COMPONENT_LABEL_VERSION_1 1 62 #define RF_COMPONENT_LABEL_VERSION 2 63 #define RF_RAID_DIRTY 0 64 #define RF_RAID_CLEAN 1 65 66 67 /* 68 * Each row in the array is a distinct parity group, so 69 * each has its own status, which is one of the following. 70 */ 71 typedef enum RF_RowStatus_e { 72 rf_rs_optimal, 73 rf_rs_degraded, 74 rf_rs_reconstructing, 75 rf_rs_reconfigured 76 } RF_RowStatus_t; 77 78 struct RF_CumulativeStats_s { 79 struct timeval start; /* the time when the stats were last started */ 80 struct timeval stop; /* the time when the stats were last stopped */ 81 long sum_io_us; /* sum of all user response times (us) */ 82 long num_ios; /* total number of I/Os serviced */ 83 long num_sect_moved; /* total number of sectors read or written */ 84 }; 85 86 struct RF_ThroughputStats_s { 87 rf_declare_mutex2(mutex);/* a mutex used to lock the configuration 88 * stuff */ 89 struct timeval start; /* timer started when numOutstandingRequests 90 * moves from 0 to 1 */ 91 struct timeval stop; /* timer stopped when numOutstandingRequests 92 * moves from 1 to 0 */ 93 RF_uint64 sum_io_us; /* total time timer is enabled */ 94 RF_uint64 num_ios; /* total number of ios processed by RAIDframe */ 95 long num_out_ios; /* number of outstanding ios */ 96 }; 97 98 struct RF_Raid_s { 99 /* This portion never changes, and can be accessed without locking */ 100 /* an exception is Disks[][].status, which requires locking when it is 101 * changed. XXX this is no longer true. numSpare and friends can 102 * change now. 103 */ 104 u_int numCol; /* number of columns of disks, typically == # 105 * of disks/rank */ 106 u_int numSpare; /* number of spare disks */ 107 int maxQueueDepth; /* max disk queue depth */ 108 RF_SectorCount_t totalSectors; /* total number of sectors in the 109 * array */ 110 RF_SectorCount_t sectorsPerDisk; /* number of sectors on each 111 * disk */ 112 u_int logBytesPerSector; /* base-2 log of the number of bytes 113 * in a sector */ 114 u_int bytesPerSector; /* bytes in a sector */ 115 RF_int32 sectorMask; /* mask of bytes-per-sector */ 116 117 RF_RaidLayout_t Layout; /* all information related to layout */ 118 RF_RaidDisk_t *Disks; /* all information related to physical disks */ 119 RF_DiskQueue_t *Queues; /* all information related to disk queues */ 120 u_int maxQueue; /* initialized queues in Queues array */ 121 const RF_DiskQueueSW_t *qType;/* pointer to the DiskQueueSW used for the 122 component queues. */ 123 /* NOTE: This is an anchor point via which the queues can be 124 * accessed, but the enqueue/dequeue routines in diskqueue.c use a 125 * local copy of this pointer for the actual accesses. */ 126 /* The remainder of the structure can change, and therefore requires 127 * locking on reads and updates */ 128 rf_declare_mutex2(mutex);/* mutex used to serialize access to 129 * the fields below */ 130 RF_RowStatus_t status; /* the status of each row in the array */ 131 int valid; /* indicates successful configuration */ 132 RF_LockTableEntry_t *lockTable; /* stripe-lock table */ 133 RF_LockTableEntry_t *quiesceLock; /* quiesnce table */ 134 int numFailures; /* total number of failures in the array */ 135 int numNewFailures; /* number of *new* failures (that havn't 136 caused a mod_counter update */ 137 138 int parity_good; /* !0 if parity is known to be correct */ 139 int serial_number; /* a "serial number" for this set */ 140 int mod_counter; /* modification counter for component labels */ 141 int clean; /* completely unused and should be removed */ 142 143 int openings; /* Number of IO's which can be scheduled 144 simultaneously (high-level - not a 145 per-component limit)*/ 146 147 int maxOutstanding; /* maxOutstanding requests (per-component) */ 148 int autoconfigure; /* automatically configure this RAID set. 149 0 == no, 1 == yes */ 150 int root_partition; /* Use this set as / 151 0 == no, 1 == yes*/ 152 int last_unit; /* last unit number (e.g. 0 for /dev/raid0) 153 of this component. Used for autoconfigure 154 only. */ 155 int config_order; /* 0 .. n. The order in which the component 156 should be auto-configured. E.g. 0 is will 157 done first, (and would become raid0). 158 This may be in conflict with last_unit!!?! */ 159 /* Not currently used. */ 160 161 /* queue to gather up requests from KernelWakeupFunc() and let 162 a kernel thread deal with calling rf_DiskIOComplete and any 163 callback functions. */ 164 TAILQ_HEAD(iodone_q,RF_DiskQueueData_s) iodone; 165 /* and a lock/cv to protect it */ 166 rf_declare_mutex2(iodone_lock); 167 rf_declare_cond2(iodone_cv); 168 169 170 RF_VoidPointerListElem_t *iobuf; /* I/O buffer free list */ 171 int iobuf_count; /* count of I/O buffers on the freelist */ 172 int numEmergencyBuffers; /* number of these buffers to pre-allocate */ 173 174 RF_VoidPointerListElem_t *stripebuf; /* Full-stripe buffer free list */ 175 int stripebuf_count; /* count of full-stripe buffers on the freelist */ 176 int numEmergencyStripeBuffers; /* number of these buffers to pre-allocate */ 177 178 /* 179 * Cleanup stuff 180 */ 181 RF_ShutdownList_t *shutdownList; /* shutdown activities */ 182 RF_AllocListElem_t *cleanupList; /* memory to be freed at 183 * shutdown time */ 184 185 /* 186 * Recon stuff 187 */ 188 RF_HeadSepLimit_t headSepLimit; 189 int numFloatingReconBufs; 190 int reconInProgress; 191 int forceRecon; 192 rf_declare_cond2(waitForReconCond); /* goes with raidPtr->mutex */ 193 RF_RaidReconDesc_t *reconDesc; /* reconstruction descriptor */ 194 RF_ReconCtrl_t *reconControl; /* reconstruction control structure 195 * pointers for each row in the array */ 196 197 /* 198 * Array-quiescence stuff 199 */ 200 rf_declare_mutex2(access_suspend_mutex); 201 rf_declare_cond2(access_suspend_cv); 202 RF_IoCount_t accesses_suspended; 203 RF_IoCount_t accs_in_flight; 204 int access_suspend_release; 205 int waiting_for_quiescence; 206 RF_CallbackFuncDesc_t *quiesce_wait_list; 207 208 /* 209 * Statistics 210 */ 211 RF_StripeCount_t parity_rewrite_stripes_done; 212 213 int recon_in_progress; 214 int parity_rewrite_in_progress; 215 int changing_components; 216 217 rf_declare_cond2(parity_rewrite_cv); 218 rf_declare_cond2(changing_components_cv); 219 220 /* 221 * Engine thread control 222 */ 223 rf_declare_mutex2(node_queue_mutex); 224 rf_declare_cond2(node_queue_cv); 225 RF_DagNode_t *node_queue; 226 RF_Thread_t parity_rewrite_thread; 227 RF_Thread_t engine_thread; 228 RF_Thread_t engine_helper_thread; 229 RF_Thread_t recon_thread; 230 int shutdown_engine; 231 int shutdown_raidio; 232 int dags_in_flight; /* debug */ 233 234 /* 235 * PSS (Parity Stripe Status) stuff 236 */ 237 long pssTableSize; 238 239 /* 240 * Reconstruction stuff 241 */ 242 int procsInBufWait; 243 int numFullReconBuffers; 244 #if RF_ACC_TRACE > 0 245 RF_AccTraceEntry_t *recon_tracerecs; 246 #endif 247 unsigned long accumXorTimeUs; 248 249 /* 250 * nAccOutstanding, waitShutdown protected by desc freelist lock 251 * (This may seem strange, since that's a central serialization point 252 * for a per-array piece of data, but otherwise, it'd be an extra 253 * per-array lock, and that'd only be less efficient...) 254 */ 255 rf_declare_mutex2(rad_lock); 256 rf_declare_cond2(outstandingCond); 257 int waitShutdown; 258 int nAccOutstanding; 259 260 int *abortRecon; /* Abort background operations requested */ 261 262 RF_DiskId_t **diskids; 263 264 int raidid; 265 void *softc; 266 RF_AccTotals_t acc_totals; 267 int keep_acc_totals; 268 269 struct raidcinfo *raid_cinfo; /* array of component info */ 270 271 int terminate_disk_queues; 272 273 /* 274 * XXX 275 * 276 * config-specific information should be moved 277 * somewhere else, or at least hung off this 278 * in some generic way 279 */ 280 #if RF_INCLUDE_CHAINDECLUSTER > 0 281 282 /* used by rf_compute_workload_shift */ 283 RF_RowCol_t hist_diskreq[RF_MAXCOL]; 284 #endif 285 /* used by declustering */ 286 int noRotate; 287 288 #if RF_INCLUDE_PARITYLOGGING > 0 289 /* used by parity logging */ 290 RF_SectorCount_t regionLogCapacity; 291 RF_ParityLogQueue_t parityLogPool; /* pool of unused parity logs */ 292 RF_RegionInfo_t *regionInfo; /* array of region state */ 293 int numParityLogs; 294 int numSectorsPerLog; 295 int regionParityRange; 296 int logsInUse; /* debugging */ 297 RF_ParityLogDiskQueue_t parityLogDiskQueue; /* state of parity 298 * logging disk work */ 299 RF_RegionBufferQueue_t regionBufferPool; /* buffers for holding 300 * region log */ 301 RF_RegionBufferQueue_t parityBufferPool; /* buffers for holding 302 * parity */ 303 void *parityLogBufferHeap; /* pool of unused parity logs */ 304 RF_Thread_t pLogDiskThreadHandle; 305 306 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */ 307 struct rf_paritymap *parity_map; 308 struct RF_Pools_s pools; 309 struct RF_PoolNames_s poolNames; 310 }; 311 312 struct raid_softc { 313 struct dk_softc sc_dksc; 314 int sc_unit; 315 int sc_flags; /* flags */ 316 int sc_cflags; /* configuration flags */ 317 kmutex_t sc_mutex; /* interlock mutex */ 318 kcondvar_t sc_cv; /* and the condvar */ 319 uint64_t sc_size; /* size of the raid device */ 320 char sc_xname[20]; /* XXX external name */ 321 RF_Raid_t sc_r; 322 LIST_ENTRY(raid_softc) sc_link; 323 }; 324 /* sc_flags */ 325 #define RAIDF_INITED 0x01 /* unit has been initialized */ 326 #define RAIDF_SHUTDOWN 0x02 /* unit is being shutdown */ 327 #define RAIDF_DETACH 0x04 /* detach after final close */ 328 #define RAIDF_WANTED 0x08 /* someone waiting to obtain a lock */ 329 #define RAIDF_LOCKED 0x10 /* unit is locked */ 330 #define RAIDF_UNIT_CHANGED 0x20 /* unit is being changed */ 331 332 333 int rf_fail_disk(RF_Raid_t *, struct rf_recon_req *); 334 335 int rf_inited(const struct raid_softc *); 336 int rf_get_unit(const struct raid_softc *); 337 RF_Raid_t *rf_get_raid(struct raid_softc *); 338 int rf_construct(struct raid_softc *, RF_Config_t *); 339 340 #endif /* !_RF__RF_RAID_H_ */ 341