1 /* $NetBSD: rf_dagfuncs.c,v 1.35 2021/08/07 16:19:15 thorpej Exp $ */
2 /*
3 * Copyright (c) 1995 Carnegie-Mellon University.
4 * All rights reserved.
5 *
6 * Author: Mark Holland, William V. Courtright II
7 *
8 * Permission to use, copy, modify and distribute this software and
9 * its documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
17 *
18 * Carnegie Mellon requests users of this software to return to
19 *
20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
21 * School of Computer Science
22 * Carnegie Mellon University
23 * Pittsburgh PA 15213-3890
24 *
25 * any improvements or extensions that they make and grant Carnegie the
26 * rights to redistribute these changes.
27 */
28
29 /*
30 * dagfuncs.c -- DAG node execution routines
31 *
32 * Rules:
33 * 1. Every DAG execution function must eventually cause node->status to
34 * get set to "good" or "bad", and "FinishNode" to be called. In the
35 * case of nodes that complete immediately (xor, NullNodeFunc, etc),
36 * the node execution function can do these two things directly. In
37 * the case of nodes that have to wait for some event (a disk read to
38 * complete, a lock to be released, etc) to occur before they can
39 * complete, this is typically achieved by having whatever module
40 * is doing the operation call GenericWakeupFunc upon completion.
41 * 2. DAG execution functions should check the status in the DAG header
42 * and NOP out their operations if the status is not "enable". However,
43 * execution functions that release resources must be sure to release
44 * them even when they NOP out the function that would use them.
45 * Functions that acquire resources should go ahead and acquire them
46 * even when they NOP, so that a downstream release node will not have
47 * to check to find out whether or not the acquire was suppressed.
48 */
49
50 #include <sys/cdefs.h>
51 __KERNEL_RCSID(0, "$NetBSD: rf_dagfuncs.c,v 1.35 2021/08/07 16:19:15 thorpej Exp $");
52
53 #include <sys/param.h>
54 #include <sys/ioctl.h>
55
56 #include "rf_archs.h"
57 #include "rf_raid.h"
58 #include "rf_dag.h"
59 #include "rf_layout.h"
60 #include "rf_etimer.h"
61 #include "rf_acctrace.h"
62 #include "rf_diskqueue.h"
63 #include "rf_dagfuncs.h"
64 #include "rf_general.h"
65 #include "rf_engine.h"
66 #include "rf_dagutils.h"
67
68 #include "rf_kintf.h"
69
70 #if RF_INCLUDE_PARITYLOGGING > 0
71 #include "rf_paritylog.h"
72 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */
73
74 void (*rf_DiskReadFunc) (RF_DagNode_t *);
75 void (*rf_DiskWriteFunc) (RF_DagNode_t *);
76 void (*rf_DiskReadUndoFunc) (RF_DagNode_t *);
77 void (*rf_DiskWriteUndoFunc) (RF_DagNode_t *);
78 void (*rf_RegularXorUndoFunc) (RF_DagNode_t *);
79 void (*rf_SimpleXorUndoFunc) (RF_DagNode_t *);
80 void (*rf_RecoveryXorUndoFunc) (RF_DagNode_t *);
81
82 /*****************************************************************************
83 * main (only) configuration routine for this module
84 ****************************************************************************/
85 int
rf_ConfigureDAGFuncs(RF_ShutdownList_t ** listp)86 rf_ConfigureDAGFuncs(RF_ShutdownList_t **listp)
87 {
88 RF_ASSERT(((sizeof(long) == 8) && RF_LONGSHIFT == 3) ||
89 ((sizeof(long) == 4) && RF_LONGSHIFT == 2));
90 rf_DiskReadFunc = rf_DiskReadFuncForThreads;
91 rf_DiskReadUndoFunc = rf_DiskUndoFunc;
92 rf_DiskWriteFunc = rf_DiskWriteFuncForThreads;
93 rf_DiskWriteUndoFunc = rf_DiskUndoFunc;
94 rf_RegularXorUndoFunc = rf_NullNodeUndoFunc;
95 rf_SimpleXorUndoFunc = rf_NullNodeUndoFunc;
96 rf_RecoveryXorUndoFunc = rf_NullNodeUndoFunc;
97 return (0);
98 }
99
100
101
102 /*****************************************************************************
103 * the execution function associated with a terminate node
104 ****************************************************************************/
105 void
rf_TerminateFunc(RF_DagNode_t * node)106 rf_TerminateFunc(RF_DagNode_t *node)
107 {
108 RF_ASSERT(node->dagHdr->numCommits == node->dagHdr->numCommitNodes);
109 node->status = rf_good;
110 rf_FinishNode(node, RF_THREAD_CONTEXT);
111 }
112
113 void
rf_TerminateUndoFunc(RF_DagNode_t * node)114 rf_TerminateUndoFunc(RF_DagNode_t *node)
115 {
116 }
117
118
119 /*****************************************************************************
120 * execution functions associated with a mirror node
121 *
122 * parameters:
123 *
124 * 0 - physical disk address of data
125 * 1 - buffer for holding read data
126 * 2 - parity stripe ID
127 * 3 - flags
128 * 4 - physical disk address of mirror (parity)
129 *
130 ****************************************************************************/
131
132 void
rf_DiskReadMirrorIdleFunc(RF_DagNode_t * node)133 rf_DiskReadMirrorIdleFunc(RF_DagNode_t *node)
134 {
135 /* select the mirror copy with the shortest queue and fill in node
136 * parameters with physical disk address */
137
138 rf_SelectMirrorDiskIdle(node);
139 rf_DiskReadFunc(node);
140 }
141
142 #if (RF_INCLUDE_CHAINDECLUSTER > 0) || (RF_INCLUDE_INTERDECLUSTER > 0) || (RF_DEBUG_VALIDATE_DAG > 0)
143 void
rf_DiskReadMirrorPartitionFunc(RF_DagNode_t * node)144 rf_DiskReadMirrorPartitionFunc(RF_DagNode_t *node)
145 {
146 /* select the mirror copy with the shortest queue and fill in node
147 * parameters with physical disk address */
148
149 rf_SelectMirrorDiskPartition(node);
150 rf_DiskReadFunc(node);
151 }
152 #endif
153
154 void
rf_DiskReadMirrorUndoFunc(RF_DagNode_t * node)155 rf_DiskReadMirrorUndoFunc(RF_DagNode_t *node)
156 {
157 }
158
159
160
161 #if RF_INCLUDE_PARITYLOGGING > 0
162 /*****************************************************************************
163 * the execution function associated with a parity log update node
164 ****************************************************************************/
165 void
rf_ParityLogUpdateFunc(RF_DagNode_t * node)166 rf_ParityLogUpdateFunc(RF_DagNode_t *node)
167 {
168 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
169 void *bf = (void *) node->params[1].p;
170 RF_ParityLogData_t *logData;
171 #if RF_ACC_TRACE > 0
172 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
173 RF_Etimer_t timer;
174 #endif
175
176 if (node->dagHdr->status == rf_enable) {
177 #if RF_ACC_TRACE > 0
178 RF_ETIMER_START(timer);
179 #endif
180 logData = rf_CreateParityLogData(RF_UPDATE, pda, bf,
181 (RF_Raid_t *) (node->dagHdr->raidPtr),
182 node->wakeFunc, node,
183 node->dagHdr->tracerec, timer);
184 if (logData)
185 rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE);
186 else {
187 #if RF_ACC_TRACE > 0
188 RF_ETIMER_STOP(timer);
189 RF_ETIMER_EVAL(timer);
190 tracerec->plog_us += RF_ETIMER_VAL_US(timer);
191 #endif
192 (node->wakeFunc) (node, ENOMEM);
193 }
194 }
195 }
196
197
198 /*****************************************************************************
199 * the execution function associated with a parity log overwrite node
200 ****************************************************************************/
201 void
rf_ParityLogOverwriteFunc(RF_DagNode_t * node)202 rf_ParityLogOverwriteFunc(RF_DagNode_t *node)
203 {
204 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
205 void *bf = (void *) node->params[1].p;
206 RF_ParityLogData_t *logData;
207 #if RF_ACC_TRACE > 0
208 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
209 RF_Etimer_t timer;
210 #endif
211
212 if (node->dagHdr->status == rf_enable) {
213 #if RF_ACC_TRACE > 0
214 RF_ETIMER_START(timer);
215 #endif
216 logData = rf_CreateParityLogData(RF_OVERWRITE, pda, bf,
217 (RF_Raid_t *) (node->dagHdr->raidPtr),
218 node->wakeFunc, node, node->dagHdr->tracerec, timer);
219 if (logData)
220 rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE);
221 else {
222 #if RF_ACC_TRACE > 0
223 RF_ETIMER_STOP(timer);
224 RF_ETIMER_EVAL(timer);
225 tracerec->plog_us += RF_ETIMER_VAL_US(timer);
226 #endif
227 (node->wakeFunc) (node, ENOMEM);
228 }
229 }
230 }
231
232 void
rf_ParityLogUpdateUndoFunc(RF_DagNode_t * node)233 rf_ParityLogUpdateUndoFunc(RF_DagNode_t *node)
234 {
235 }
236
237 void
rf_ParityLogOverwriteUndoFunc(RF_DagNode_t * node)238 rf_ParityLogOverwriteUndoFunc(RF_DagNode_t *node)
239 {
240 }
241 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */
242
243 /*****************************************************************************
244 * the execution function associated with a NOP node
245 ****************************************************************************/
246 void
rf_NullNodeFunc(RF_DagNode_t * node)247 rf_NullNodeFunc(RF_DagNode_t *node)
248 {
249 node->status = rf_good;
250 rf_FinishNode(node, RF_THREAD_CONTEXT);
251 }
252
253 void
rf_NullNodeUndoFunc(RF_DagNode_t * node)254 rf_NullNodeUndoFunc(RF_DagNode_t *node)
255 {
256 node->status = rf_undone;
257 rf_FinishNode(node, RF_THREAD_CONTEXT);
258 }
259
260
261 /*****************************************************************************
262 * the execution function associated with a disk-read node
263 ****************************************************************************/
264 void
rf_DiskReadFuncForThreads(RF_DagNode_t * node)265 rf_DiskReadFuncForThreads(RF_DagNode_t *node)
266 {
267 RF_DiskQueueData_t *req;
268 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
269 void *bf = (void *) node->params[1].p;
270 RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v;
271 unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v);
272 unsigned which_ru = RF_EXTRACT_RU(node->params[3].v);
273 RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_READ : RF_IO_TYPE_NOP;
274 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
275
276 req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector,
277 bf, parityStripeID, which_ru, node->wakeFunc, node,
278 #if RF_ACC_TRACE > 0
279 node->dagHdr->tracerec,
280 #else
281 NULL,
282 #endif
283 (void *) (node->dagHdr->raidPtr), 0, node->dagHdr->bp);
284
285 node->dagFuncData = (void *) req;
286 rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority);
287 }
288
289
290 /*****************************************************************************
291 * the execution function associated with a disk-write node
292 ****************************************************************************/
293 void
rf_DiskWriteFuncForThreads(RF_DagNode_t * node)294 rf_DiskWriteFuncForThreads(RF_DagNode_t *node)
295 {
296 RF_DiskQueueData_t *req;
297 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
298 void *bf = (void *) node->params[1].p;
299 RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v;
300 unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v);
301 unsigned which_ru = RF_EXTRACT_RU(node->params[3].v);
302 RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_WRITE : RF_IO_TYPE_NOP;
303 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
304
305 /* normal processing (rollaway or forward recovery) begins here */
306 req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector,
307 bf, parityStripeID, which_ru, node->wakeFunc, node,
308 #if RF_ACC_TRACE > 0
309 node->dagHdr->tracerec,
310 #else
311 NULL,
312 #endif
313 (void *) (node->dagHdr->raidPtr),
314 0, node->dagHdr->bp);
315
316 node->dagFuncData = (void *) req;
317 rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority);
318 }
319 /*****************************************************************************
320 * the undo function for disk nodes
321 * Note: this is not a proper undo of a write node, only locks are released.
322 * old data is not restored to disk!
323 ****************************************************************************/
324 void
rf_DiskUndoFunc(RF_DagNode_t * node)325 rf_DiskUndoFunc(RF_DagNode_t *node)
326 {
327 RF_DiskQueueData_t *req;
328 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
329 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
330
331 req = rf_CreateDiskQueueData(RF_IO_TYPE_NOP,
332 0L, 0, NULL, 0L, 0, node->wakeFunc, node,
333 #if RF_ACC_TRACE > 0
334 node->dagHdr->tracerec,
335 #else
336 NULL,
337 #endif
338 (void *) (node->dagHdr->raidPtr),
339 0, NULL);
340
341 node->dagFuncData = (void *) req;
342 rf_DiskIOEnqueue(&(dqs[pda->col]), req, RF_IO_NORMAL_PRIORITY);
343 }
344
345 /*****************************************************************************
346 * Callback routine for DiskRead and DiskWrite nodes. When the disk
347 * op completes, the routine is called to set the node status and
348 * inform the execution engine that the node has fired.
349 ****************************************************************************/
350 void
rf_GenericWakeupFunc(void * v,int status)351 rf_GenericWakeupFunc(void *v, int status)
352 {
353 RF_DagNode_t *node = v;
354
355 switch (node->status) {
356 case rf_fired:
357 if (status)
358 node->status = rf_bad;
359 else
360 node->status = rf_good;
361 break;
362 case rf_recover:
363 /* probably should never reach this case */
364 if (status)
365 node->status = rf_panic;
366 else
367 node->status = rf_undone;
368 break;
369 default:
370 printf("rf_GenericWakeupFunc:");
371 printf("node->status is %d,", node->status);
372 printf("status is %d \n", status);
373 RF_PANIC();
374 break;
375 }
376 if (node->dagFuncData)
377 rf_FreeDiskQueueData((RF_DiskQueueData_t *) node->dagFuncData);
378 rf_FinishNode(node, RF_INTR_CONTEXT);
379 }
380
381
382 /*****************************************************************************
383 * there are three distinct types of xor nodes:
384
385 * A "regular xor" is used in the fault-free case where the access
386 * spans a complete stripe unit. It assumes that the result buffer is
387 * one full stripe unit in size, and uses the stripe-unit-offset
388 * values that it computes from the PDAs to determine where within the
389 * stripe unit to XOR each argument buffer.
390 *
391 * A "simple xor" is used in the fault-free case where the access
392 * touches only a portion of one (or two, in some cases) stripe
393 * unit(s). It assumes that all the argument buffers are of the same
394 * size and have the same stripe unit offset.
395 *
396 * A "recovery xor" is used in the degraded-mode case. It's similar
397 * to the regular xor function except that it takes the failed PDA as
398 * an additional parameter, and uses it to determine what portions of
399 * the argument buffers need to be xor'd into the result buffer, and
400 * where in the result buffer they should go.
401 ****************************************************************************/
402
403 /* xor the params together and store the result in the result field.
404 * assume the result field points to a buffer that is the size of one
405 * SU, and use the pda params to determine where within the buffer to
406 * XOR the input buffers. */
407 void
rf_RegularXorFunc(RF_DagNode_t * node)408 rf_RegularXorFunc(RF_DagNode_t *node)
409 {
410 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
411 #if RF_ACC_TRACE > 0
412 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
413 RF_Etimer_t timer;
414 #endif
415 int i, retcode;
416
417 retcode = 0;
418 if (node->dagHdr->status == rf_enable) {
419 /* don't do the XOR if the input is the same as the output */
420 #if RF_ACC_TRACE > 0
421 RF_ETIMER_START(timer);
422 #endif
423 for (i = 0; i < node->numParams - 1; i += 2)
424 if (node->params[i + 1].p != node->results[0]) {
425 retcode = rf_XorIntoBuffer(raidPtr, (RF_PhysDiskAddr_t *) node->params[i].p,
426 (char *) node->params[i + 1].p, (char *) node->results[0]);
427 }
428 #if RF_ACC_TRACE > 0
429 RF_ETIMER_STOP(timer);
430 RF_ETIMER_EVAL(timer);
431 tracerec->xor_us += RF_ETIMER_VAL_US(timer);
432 #endif
433 }
434 rf_GenericWakeupFunc(node, retcode); /* call wake func
435 * explicitly since no
436 * I/O in this node */
437 }
438 /* xor the inputs into the result buffer, ignoring placement issues */
439 void
rf_SimpleXorFunc(RF_DagNode_t * node)440 rf_SimpleXorFunc(RF_DagNode_t *node)
441 {
442 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
443 int i, retcode = 0;
444 #if RF_ACC_TRACE > 0
445 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
446 RF_Etimer_t timer;
447 #endif
448
449 if (node->dagHdr->status == rf_enable) {
450 #if RF_ACC_TRACE > 0
451 RF_ETIMER_START(timer);
452 #endif
453 /* don't do the XOR if the input is the same as the output */
454 for (i = 0; i < node->numParams - 1; i += 2)
455 if (node->params[i + 1].p != node->results[0]) {
456 retcode = rf_bxor((char *) node->params[i + 1].p, (char *) node->results[0],
457 rf_RaidAddressToByte(raidPtr, ((RF_PhysDiskAddr_t *) node->params[i].p)->numSector));
458 }
459 #if RF_ACC_TRACE > 0
460 RF_ETIMER_STOP(timer);
461 RF_ETIMER_EVAL(timer);
462 tracerec->xor_us += RF_ETIMER_VAL_US(timer);
463 #endif
464 }
465 rf_GenericWakeupFunc(node, retcode); /* call wake func
466 * explicitly since no
467 * I/O in this node */
468 }
469 /* this xor is used by the degraded-mode dag functions to recover lost
470 * data. the second-to-last parameter is the PDA for the failed
471 * portion of the access. the code here looks at this PDA and assumes
472 * that the xor target buffer is equal in size to the number of
473 * sectors in the failed PDA. It then uses the other PDAs in the
474 * parameter list to determine where within the target buffer the
475 * corresponding data should be xored. */
476 void
rf_RecoveryXorFunc(RF_DagNode_t * node)477 rf_RecoveryXorFunc(RF_DagNode_t *node)
478 {
479 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
480 RF_RaidLayout_t *layoutPtr = (RF_RaidLayout_t *) & raidPtr->Layout;
481 RF_PhysDiskAddr_t *failedPDA = (RF_PhysDiskAddr_t *) node->params[node->numParams - 2].p;
482 int i, retcode = 0;
483 RF_PhysDiskAddr_t *pda;
484 int suoffset, failedSUOffset = rf_StripeUnitOffset(layoutPtr, failedPDA->startSector);
485 char *srcbuf, *destbuf;
486 #if RF_ACC_TRACE > 0
487 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
488 RF_Etimer_t timer;
489 #endif
490
491 if (node->dagHdr->status == rf_enable) {
492 #if RF_ACC_TRACE > 0
493 RF_ETIMER_START(timer);
494 #endif
495 for (i = 0; i < node->numParams - 2; i += 2)
496 if (node->params[i + 1].p != node->results[0]) {
497 pda = (RF_PhysDiskAddr_t *) node->params[i].p;
498 srcbuf = (char *) node->params[i + 1].p;
499 suoffset = rf_StripeUnitOffset(layoutPtr, pda->startSector);
500 destbuf = ((char *) node->results[0]) + rf_RaidAddressToByte(raidPtr, suoffset - failedSUOffset);
501 retcode = rf_bxor(srcbuf, destbuf, rf_RaidAddressToByte(raidPtr, pda->numSector));
502 }
503 #if RF_ACC_TRACE > 0
504 RF_ETIMER_STOP(timer);
505 RF_ETIMER_EVAL(timer);
506 tracerec->xor_us += RF_ETIMER_VAL_US(timer);
507 #endif
508 }
509 rf_GenericWakeupFunc(node, retcode);
510 }
511 /*****************************************************************************
512 * The next three functions are utilities used by the above
513 * xor-execution functions.
514 ****************************************************************************/
515
516
517 /*
518 * this is just a glorified buffer xor. targbuf points to a buffer
519 * that is one full stripe unit in size. srcbuf points to a buffer
520 * that may be less than 1 SU, but never more. When the access
521 * described by pda is one SU in size (which by implication means it's
522 * SU-aligned), all that happens is (targbuf) <- (srcbuf ^ targbuf).
523 * When the access is less than one SU in size the XOR occurs on only
524 * the portion of targbuf identified in the pda. */
525
526 int
rf_XorIntoBuffer(RF_Raid_t * raidPtr,RF_PhysDiskAddr_t * pda,char * srcbuf,char * targbuf)527 rf_XorIntoBuffer(RF_Raid_t *raidPtr, RF_PhysDiskAddr_t *pda,
528 char *srcbuf, char *targbuf)
529 {
530 char *targptr;
531 int sectPerSU = raidPtr->Layout.sectorsPerStripeUnit;
532 int SUOffset = pda->startSector % sectPerSU;
533 int length, retcode = 0;
534
535 RF_ASSERT(pda->numSector <= sectPerSU);
536
537 targptr = targbuf + rf_RaidAddressToByte(raidPtr, SUOffset);
538 length = rf_RaidAddressToByte(raidPtr, pda->numSector);
539 retcode = rf_bxor(srcbuf, targptr, length);
540 return (retcode);
541 }
542 /* it really should be the case that the buffer pointers (returned by
543 * malloc) are aligned to the natural word size of the machine, so
544 * this is the only case we optimize for. The length should always be
545 * a multiple of the sector size, so there should be no problem with
546 * leftover bytes at the end. */
547 int
rf_bxor(char * src,char * dest,int len)548 rf_bxor(char *src, char *dest, int len)
549 {
550 unsigned mask = sizeof(long) - 1, retcode = 0;
551
552 if (!(((unsigned long) src) & mask) &&
553 !(((unsigned long) dest) & mask) && !(len & mask)) {
554 retcode = rf_longword_bxor((unsigned long *) src,
555 (unsigned long *) dest,
556 len >> RF_LONGSHIFT);
557 } else {
558 RF_ASSERT(0);
559 }
560 return (retcode);
561 }
562
563 /* When XORing in kernel mode, we need to map each user page to kernel
564 * space before we can access it. We don't want to assume anything
565 * about which input buffers are in kernel/user space, nor about their
566 * alignment, so in each loop we compute the maximum number of bytes
567 * that we can xor without crossing any page boundaries, and do only
568 * this many bytes before the next remap.
569 *
570 * len - is in longwords
571 */
572 int
rf_longword_bxor(unsigned long * src,unsigned long * dest,int len)573 rf_longword_bxor(unsigned long *src, unsigned long *dest, int len)
574 {
575 unsigned long *end = src + len;
576 unsigned long d0, d1, d2, d3, s0, s1, s2, s3; /* temps */
577 unsigned long *pg_src, *pg_dest; /* per-page source/dest pointers */
578 int longs_this_time;/* # longwords to xor in the current iteration */
579
580 pg_src = src;
581 pg_dest = dest;
582 if (!pg_src || !pg_dest)
583 return (EFAULT);
584
585 while (len >= 4) {
586 longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(pg_src), RF_BLIP(pg_dest)) >> RF_LONGSHIFT); /* note len in longwords */
587 src += longs_this_time;
588 dest += longs_this_time;
589 len -= longs_this_time;
590 while (longs_this_time >= 4) {
591 d0 = pg_dest[0];
592 d1 = pg_dest[1];
593 d2 = pg_dest[2];
594 d3 = pg_dest[3];
595 s0 = pg_src[0];
596 s1 = pg_src[1];
597 s2 = pg_src[2];
598 s3 = pg_src[3];
599 pg_dest[0] = d0 ^ s0;
600 pg_dest[1] = d1 ^ s1;
601 pg_dest[2] = d2 ^ s2;
602 pg_dest[3] = d3 ^ s3;
603 pg_src += 4;
604 pg_dest += 4;
605 longs_this_time -= 4;
606 }
607 while (longs_this_time > 0) { /* cannot cross any page
608 * boundaries here */
609 *pg_dest++ ^= *pg_src++;
610 longs_this_time--;
611 }
612
613 /* either we're done, or we've reached a page boundary on one
614 * (or possibly both) of the pointers */
615 if (len) {
616 if (RF_PAGE_ALIGNED(src))
617 pg_src = src;
618 if (RF_PAGE_ALIGNED(dest))
619 pg_dest = dest;
620 if (!pg_src || !pg_dest)
621 return (EFAULT);
622 }
623 }
624 while (src < end) {
625 *pg_dest++ ^= *pg_src++;
626 src++;
627 dest++;
628 len--;
629 if (RF_PAGE_ALIGNED(src))
630 pg_src = src;
631 if (RF_PAGE_ALIGNED(dest))
632 pg_dest = dest;
633 }
634 RF_ASSERT(len == 0);
635 return (0);
636 }
637
638 #if 0
639 /*
640 dst = a ^ b ^ c;
641 a may equal dst
642 see comment above longword_bxor
643 len is length in longwords
644 */
645 int
646 rf_longword_bxor3(unsigned long *dst, unsigned long *a, unsigned long *b,
647 unsigned long *c, int len, void *bp)
648 {
649 unsigned long a0, a1, a2, a3, b0, b1, b2, b3;
650 unsigned long *pg_a, *pg_b, *pg_c, *pg_dst; /* per-page source/dest
651 * pointers */
652 int longs_this_time;/* # longs to xor in the current iteration */
653 char dst_is_a = 0;
654
655 pg_a = a;
656 pg_b = b;
657 pg_c = c;
658 if (a == dst) {
659 pg_dst = pg_a;
660 dst_is_a = 1;
661 } else {
662 pg_dst = dst;
663 }
664
665 /* align dest to cache line. Can't cross a pg boundary on dst here. */
666 while ((((unsigned long) pg_dst) & 0x1f)) {
667 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
668 dst++;
669 a++;
670 b++;
671 c++;
672 if (RF_PAGE_ALIGNED(a)) {
673 pg_a = a;
674 if (!pg_a)
675 return (EFAULT);
676 }
677 if (RF_PAGE_ALIGNED(b)) {
678 pg_b = a;
679 if (!pg_b)
680 return (EFAULT);
681 }
682 if (RF_PAGE_ALIGNED(c)) {
683 pg_c = a;
684 if (!pg_c)
685 return (EFAULT);
686 }
687 len--;
688 }
689
690 while (len > 4) {
691 longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(a), RF_MIN(RF_BLIP(b), RF_MIN(RF_BLIP(c), RF_BLIP(dst)))) >> RF_LONGSHIFT);
692 a += longs_this_time;
693 b += longs_this_time;
694 c += longs_this_time;
695 dst += longs_this_time;
696 len -= longs_this_time;
697 while (longs_this_time >= 4) {
698 a0 = pg_a[0];
699 longs_this_time -= 4;
700
701 a1 = pg_a[1];
702 a2 = pg_a[2];
703
704 a3 = pg_a[3];
705 pg_a += 4;
706
707 b0 = pg_b[0];
708 b1 = pg_b[1];
709
710 b2 = pg_b[2];
711 b3 = pg_b[3];
712 /* start dual issue */
713 a0 ^= b0;
714 b0 = pg_c[0];
715
716 pg_b += 4;
717 a1 ^= b1;
718
719 a2 ^= b2;
720 a3 ^= b3;
721
722 b1 = pg_c[1];
723 a0 ^= b0;
724
725 b2 = pg_c[2];
726 a1 ^= b1;
727
728 b3 = pg_c[3];
729 a2 ^= b2;
730
731 pg_dst[0] = a0;
732 a3 ^= b3;
733 pg_dst[1] = a1;
734 pg_c += 4;
735 pg_dst[2] = a2;
736 pg_dst[3] = a3;
737 pg_dst += 4;
738 }
739 while (longs_this_time > 0) { /* cannot cross any page
740 * boundaries here */
741 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
742 longs_this_time--;
743 }
744
745 if (len) {
746 if (RF_PAGE_ALIGNED(a)) {
747 pg_a = a;
748 if (!pg_a)
749 return (EFAULT);
750 if (dst_is_a)
751 pg_dst = pg_a;
752 }
753 if (RF_PAGE_ALIGNED(b)) {
754 pg_b = b;
755 if (!pg_b)
756 return (EFAULT);
757 }
758 if (RF_PAGE_ALIGNED(c)) {
759 pg_c = c;
760 if (!pg_c)
761 return (EFAULT);
762 }
763 if (!dst_is_a)
764 if (RF_PAGE_ALIGNED(dst)) {
765 pg_dst = dst;
766 if (!pg_dst)
767 return (EFAULT);
768 }
769 }
770 }
771 while (len) {
772 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
773 dst++;
774 a++;
775 b++;
776 c++;
777 if (RF_PAGE_ALIGNED(a)) {
778 pg_a = a;
779 if (!pg_a)
780 return (EFAULT);
781 if (dst_is_a)
782 pg_dst = pg_a;
783 }
784 if (RF_PAGE_ALIGNED(b)) {
785 pg_b = b;
786 if (!pg_b)
787 return (EFAULT);
788 }
789 if (RF_PAGE_ALIGNED(c)) {
790 pg_c = c;
791 if (!pg_c)
792 return (EFAULT);
793 }
794 if (!dst_is_a)
795 if (RF_PAGE_ALIGNED(dst)) {
796 pg_dst = dst;
797 if (!pg_dst)
798 return (EFAULT);
799 }
800 len--;
801 }
802 return (0);
803 }
804
805 int
806 rf_bxor3(unsigned char *dst, unsigned char *a, unsigned char *b,
807 unsigned char *c, unsigned long len, void *bp)
808 {
809 RF_ASSERT(((RF_UL(dst) | RF_UL(a) | RF_UL(b) | RF_UL(c) | len) & 0x7) == 0);
810
811 return (rf_longword_bxor3((unsigned long *) dst, (unsigned long *) a,
812 (unsigned long *) b, (unsigned long *) c, len >> RF_LONGSHIFT, bp));
813 }
814 #endif
815