1 /*
2 * Copyright (c) 2012 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/proc.h>
38 #include <sys/sysctl.h>
39 #include <sys/buf.h>
40 #include <sys/conf.h>
41 #include <sys/disklabel.h>
42 #include <sys/disklabel32.h>
43 #include <sys/disklabel64.h>
44 #include <sys/diskslice.h>
45 #include <sys/diskmbr.h>
46 #include <sys/disk.h>
47 #include <sys/malloc.h>
48 #include <sys/device.h>
49 #include <sys/devfs.h>
50 #include <sys/thread.h>
51 #include <sys/queue.h>
52 #include <sys/lock.h>
53 #include <sys/stat.h>
54 #include <sys/uuid.h>
55 #include <sys/dmsg.h>
56
57 #include <sys/buf2.h>
58 #include <sys/msgport2.h>
59
60 struct dios_open {
61 int openrd;
62 int openwr;
63 };
64
65 struct dios_io {
66 int count;
67 int eof;
68 kdmsg_data_t data;
69 };
70
71 static MALLOC_DEFINE(M_DMSG_DISK, "dmsg_disk", "disk dmsg");
72
73 static int blk_active;
74 SYSCTL_INT(_debug, OID_AUTO, blk_active, CTLFLAG_RW, &blk_active, 0,
75 "Number of active iocom IOs");
76
77 static int disk_iocom_reconnect(struct disk *dp, struct file *fp);
78 static int disk_rcvdmsg(kdmsg_msg_t *msg);
79
80 static void disk_blk_open(struct disk *dp, kdmsg_msg_t *msg);
81 static void disk_blk_read(struct disk *dp, kdmsg_msg_t *msg);
82 static void disk_blk_write(struct disk *dp, kdmsg_msg_t *msg);
83 static void disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg);
84 static void disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg);
85 static void diskiodone(struct bio *bio);
86
87 void
disk_iocom_init(struct disk * dp)88 disk_iocom_init(struct disk *dp)
89 {
90 kdmsg_iocom_init(&dp->d_iocom, dp,
91 KDMSG_IOCOMF_AUTOCONN |
92 KDMSG_IOCOMF_AUTORXSPAN |
93 KDMSG_IOCOMF_AUTOTXSPAN,
94 M_DMSG_DISK, disk_rcvdmsg);
95 }
96
97 void
disk_iocom_update(struct disk * dp)98 disk_iocom_update(struct disk *dp)
99 {
100 }
101
102 void
disk_iocom_uninit(struct disk * dp)103 disk_iocom_uninit(struct disk *dp)
104 {
105 kdmsg_iocom_uninit(&dp->d_iocom);
106 }
107
108 int
disk_iocom_ioctl(struct disk * dp,u_long cmd,void * data)109 disk_iocom_ioctl(struct disk *dp, u_long cmd, void *data)
110 {
111 struct file *fp;
112 struct disk_ioc_recluster *recl;
113 int error;
114
115 switch(cmd) {
116 case DIOCRECLUSTER:
117 recl = data;
118 fp = holdfp(curthread, recl->fd, -1);
119 if (fp) {
120 error = disk_iocom_reconnect(dp, fp);
121 } else {
122 error = EINVAL;
123 }
124 break;
125 default:
126 error = EOPNOTSUPP;
127 break;
128 }
129 return error;
130 }
131
132 static
133 int
disk_iocom_reconnect(struct disk * dp,struct file * fp)134 disk_iocom_reconnect(struct disk *dp, struct file *fp)
135 {
136 char devname[64];
137
138 ksnprintf(devname, sizeof(devname), "%s%d",
139 dev_dname(dp->d_rawdev), dkunit(dp->d_rawdev));
140
141 kdmsg_iocom_reconnect(&dp->d_iocom, fp, devname);
142
143 dp->d_iocom.auto_lnk_conn.proto_version = DMSG_SPAN_PROTO_1;
144 dp->d_iocom.auto_lnk_conn.peer_type = DMSG_PEER_BLOCK;
145 dp->d_iocom.auto_lnk_conn.peer_mask = 1LLU << DMSG_PEER_BLOCK;
146 dp->d_iocom.auto_lnk_conn.peer_mask = (uint64_t)-1;
147 #if 0
148 if (dp->d_info.d_serialno) {
149 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
150 sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
151 "%s/%s", hostname, dp->d_info.d_serialno);
152 } else {
153 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
154 sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
155 "%s/%s", hostname, devname);
156 }
157 #endif
158 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
159 sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
160 "%s/%s", hostname, devname);
161
162 dp->d_iocom.auto_lnk_span.proto_version = DMSG_SPAN_PROTO_1;
163 dp->d_iocom.auto_lnk_span.peer_type = DMSG_PEER_BLOCK;
164 dp->d_iocom.auto_lnk_span.media.block.bytes =
165 dp->d_info.d_media_size;
166 dp->d_iocom.auto_lnk_span.media.block.blksize =
167 dp->d_info.d_media_blksize;
168 ksnprintf(dp->d_iocom.auto_lnk_span.peer_label,
169 sizeof(dp->d_iocom.auto_lnk_span.peer_label),
170 "%s", dp->d_iocom.auto_lnk_conn.peer_label);
171 if (dp->d_info.d_serialno) {
172 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label,
173 sizeof(dp->d_iocom.auto_lnk_span.pfs_label),
174 "%s", dp->d_info.d_serialno);
175 } else {
176 /*
177 * If no serial number is available generate a dummy serial
178 * number from the host and device name and pray. This will
179 * allow e.g. /dev/vn* to look meaningful on a remote machine.
180 */
181 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label,
182 sizeof(dp->d_iocom.auto_lnk_span.pfs_label),
183 "%s.%s", hostname, devname);
184 }
185
186 kdmsg_iocom_autoinitiate(&dp->d_iocom, NULL);
187
188 return (0);
189 }
190
191 static int
disk_rcvdmsg(kdmsg_msg_t * msg)192 disk_rcvdmsg(kdmsg_msg_t *msg)
193 {
194 struct disk *dp = msg->state->iocom->handle;
195
196 /*
197 * Handle debug messages (these might not be in transactions)
198 */
199 switch(msg->any.head.cmd & DMSGF_CMDSWMASK) {
200 case DMSG_DBG_SHELL:
201 /*
202 * Execute shell command (not supported atm)
203 */
204 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
205 return(0);
206 case DMSG_DBG_SHELL | DMSGF_REPLY:
207 if (msg->aux_data) {
208 msg->aux_data[msg->aux_size - 1] = 0;
209 kprintf("diskiocom: DEBUGMSG: %s\n", msg->aux_data);
210 }
211 return(0);
212 }
213
214 /*
215 * All remaining messages must be in a transaction.
216 *
217 * NOTE! We currently don't care if the transaction is just
218 * the span transaction (for disk probes) or if it is the
219 * BLK_OPEN transaction.
220 *
221 * NOTE! We are switching on the first message's command. The
222 * actual message command within the transaction may be
223 * different (if streaming within a transaction).
224 */
225 if (msg->state == &msg->state->iocom->state0) {
226 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
227 return(0);
228 }
229
230 switch(msg->state->rxcmd & DMSGF_CMDSWMASK) {
231 case DMSG_BLK_OPEN:
232 disk_blk_open(dp, msg);
233 break;
234 case DMSG_BLK_READ:
235 /*
236 * not reached normally but leave in for completeness
237 */
238 disk_blk_read(dp, msg);
239 break;
240 case DMSG_BLK_WRITE:
241 disk_blk_write(dp, msg);
242 break;
243 case DMSG_BLK_FLUSH:
244 disk_blk_flush(dp, msg);
245 break;
246 case DMSG_BLK_FREEBLKS:
247 disk_blk_freeblks(dp, msg);
248 break;
249 default:
250 if ((msg->any.head.cmd & DMSGF_REPLY) == 0) {
251 if (msg->any.head.cmd & DMSGF_DELETE)
252 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
253 else
254 kdmsg_msg_result(msg, DMSG_ERR_NOSUPP);
255 }
256 break;
257 }
258 return (0);
259 }
260
261 static
262 void
disk_blk_open(struct disk * dp,kdmsg_msg_t * msg)263 disk_blk_open(struct disk *dp, kdmsg_msg_t *msg)
264 {
265 struct dios_open *openst;
266 int error = DMSG_ERR_NOSUPP;
267 int fflags;
268
269 openst = msg->state->any.any;
270 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_OPEN) {
271 if (openst == NULL) {
272 openst = kmalloc(sizeof(*openst), M_DEVBUF,
273 M_WAITOK | M_ZERO);
274 msg->state->any.any = openst;
275 }
276 fflags = 0;
277 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
278 fflags = FREAD;
279 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
280 fflags |= FWRITE;
281 error = dev_dopen(dp->d_rawdev, fflags, S_IFCHR,
282 proc0.p_ucred, NULL, NULL);
283 if (error) {
284 error = DMSG_ERR_IO;
285 } else {
286 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
287 ++openst->openrd;
288 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
289 ++openst->openwr;
290 }
291 }
292 #if 0
293 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_CLOSE &&
294 openst) {
295 fflags = 0;
296 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_RD) &&
297 openst->openrd) {
298 fflags = FREAD;
299 }
300 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_WR) &&
301 openst->openwr) {
302 fflags |= FWRITE;
303 }
304 error = dev_dclose(dp->d_rawdev, fflags, S_IFCHR, NULL);
305 if (error) {
306 error = DMSG_ERR_IO;
307 } else {
308 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
309 --openst->openrd;
310 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
311 --openst->openwr;
312 }
313 }
314 #endif
315 if (msg->any.head.cmd & DMSGF_DELETE) {
316 if (openst) {
317 while (openst->openrd && openst->openwr) {
318 --openst->openrd;
319 --openst->openwr;
320 dev_dclose(dp->d_rawdev, FREAD|FWRITE, S_IFCHR, NULL);
321 }
322 while (openst->openrd) {
323 --openst->openrd;
324 dev_dclose(dp->d_rawdev, FREAD, S_IFCHR, NULL);
325 }
326 while (openst->openwr) {
327 --openst->openwr;
328 dev_dclose(dp->d_rawdev, FWRITE, S_IFCHR, NULL);
329 }
330 kfree(openst, M_DEVBUF);
331 msg->state->any.any = NULL;
332 }
333 kdmsg_msg_reply(msg, error);
334 } else {
335 kdmsg_msg_result(msg, error);
336 }
337 }
338
339 static
340 void
disk_blk_read(struct disk * dp,kdmsg_msg_t * msg)341 disk_blk_read(struct disk *dp, kdmsg_msg_t *msg)
342 {
343 struct dios_io *iost;
344 struct buf *bp;
345 struct bio *bio;
346 int error = DMSG_ERR_NOSUPP;
347 int reterr = 1;
348
349 /*
350 * Only DMSG_BLK_READ commands imply read ops.
351 */
352 iost = msg->state->any.any;
353 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_READ) {
354 if (msg->any.blk_read.bytes < DEV_BSIZE ||
355 msg->any.blk_read.bytes > MAXPHYS) {
356 error = DMSG_ERR_PARAM;
357 goto done;
358 }
359 if (iost == NULL) {
360 iost = kmalloc(sizeof(*iost), M_DEVBUF,
361 M_WAITOK | M_ZERO);
362 msg->state->any.any = iost;
363 }
364 reterr = 0;
365 bp = getpbuf_mem(NULL);
366 KKASSERT(msg->any.blk_read.bytes <= bp->b_bufsize);
367 bio = &bp->b_bio1;
368 bp->b_cmd = BUF_CMD_READ;
369 bp->b_bcount = msg->any.blk_read.bytes;
370 bp->b_resid = bp->b_bcount;
371 bio->bio_offset = msg->any.blk_read.offset;
372 bio->bio_caller_info1.ptr = msg->state;
373 bio->bio_done = diskiodone;
374
375 /* kdmsg_state_hold(msg->state); */
376 atomic_add_int(&blk_active, 1);
377 atomic_add_int(&iost->count, 1);
378 if (msg->any.head.cmd & DMSGF_DELETE)
379 iost->eof = 1;
380 BUF_KERNPROC(bp);
381 dev_dstrategy(dp->d_rawdev, bio);
382 }
383 done:
384 if (reterr) {
385 if (msg->any.head.cmd & DMSGF_DELETE) {
386 if (iost && iost->count == 0) {
387 kfree(iost, M_DEVBUF);
388 msg->state->any.any = NULL;
389 }
390 kdmsg_msg_reply(msg, error);
391 } else {
392 kdmsg_msg_result(msg, error);
393 }
394 }
395 }
396
397 static
398 void
disk_blk_write(struct disk * dp,kdmsg_msg_t * msg)399 disk_blk_write(struct disk *dp, kdmsg_msg_t *msg)
400 {
401 struct dios_io *iost;
402 struct buf *bp;
403 struct bio *bio;
404 int error = DMSG_ERR_NOSUPP;
405 int reterr = 1;
406
407 /*
408 * Only DMSG_BLK_WRITE commands imply read ops.
409 */
410 iost = msg->state->any.any;
411 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_WRITE) {
412 if (msg->any.blk_write.bytes < DEV_BSIZE ||
413 msg->any.blk_write.bytes > MAXPHYS) {
414 error = DMSG_ERR_PARAM;
415 goto done;
416 }
417 if (iost == NULL) {
418 iost = kmalloc(sizeof(*iost), M_DEVBUF,
419 M_WAITOK | M_ZERO);
420 msg->state->any.any = iost;
421 }
422
423 /*
424 * Issue WRITE. Short data implies zeros. Try to optimize
425 * the buffer cache buffer for the case where we can just
426 * use the message's data pointer.
427 */
428 reterr = 0;
429 if (msg->aux_size >= msg->any.blk_write.bytes)
430 bp = getpbuf(NULL);
431 else
432 bp = getpbuf_mem(NULL);
433 KKASSERT(msg->any.blk_write.bytes <= bp->b_bufsize);
434 bio = &bp->b_bio1;
435 bp->b_cmd = BUF_CMD_WRITE;
436 bp->b_bcount = msg->any.blk_write.bytes;
437 bp->b_resid = bp->b_bcount;
438 if (msg->aux_size >= msg->any.blk_write.bytes) {
439 bp->b_data = msg->aux_data;
440 kdmsg_detach_aux_data(msg, &iost->data);
441 } else {
442 bcopy(msg->aux_data, bp->b_data, msg->aux_size);
443 bzero(bp->b_data + msg->aux_size,
444 msg->any.blk_write.bytes - msg->aux_size);
445 bzero(&iost->data, sizeof(iost->data));
446 }
447 bio->bio_offset = msg->any.blk_write.offset;
448 bio->bio_caller_info1.ptr = msg->state;
449 bio->bio_done = diskiodone;
450
451 /* kdmsg_state_hold(msg->state); */
452 atomic_add_int(&blk_active, 1);
453 atomic_add_int(&iost->count, 1);
454 if (msg->any.head.cmd & DMSGF_DELETE)
455 iost->eof = 1;
456 BUF_KERNPROC(bp);
457 dev_dstrategy(dp->d_rawdev, bio);
458 }
459 done:
460 if (reterr) {
461 if (msg->any.head.cmd & DMSGF_DELETE) {
462 if (iost && iost->count == 0) {
463 kfree(iost, M_DEVBUF);
464 msg->state->any.any = NULL;
465 }
466 kdmsg_msg_reply(msg, error);
467 } else {
468 kdmsg_msg_result(msg, error);
469 }
470 }
471 }
472
473 static
474 void
disk_blk_flush(struct disk * dp,kdmsg_msg_t * msg)475 disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg)
476 {
477 struct dios_io *iost;
478 struct buf *bp;
479 struct bio *bio;
480 int error = DMSG_ERR_NOSUPP;
481 int reterr = 1;
482
483 /*
484 * Only DMSG_BLK_FLUSH commands imply read ops.
485 */
486 iost = msg->state->any.any;
487 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FLUSH) {
488 if (iost == NULL) {
489 iost = kmalloc(sizeof(*iost), M_DEVBUF,
490 M_WAITOK | M_ZERO);
491 msg->state->any.any = iost;
492 }
493 reterr = 0;
494 bp = getpbuf(NULL);
495 bio = &bp->b_bio1;
496 bp->b_cmd = BUF_CMD_FLUSH;
497 bp->b_bcount = msg->any.blk_flush.bytes;
498 bp->b_resid = 0;
499 bio->bio_offset = msg->any.blk_flush.offset;
500 bio->bio_caller_info1.ptr = msg->state;
501 bio->bio_done = diskiodone;
502
503 /* kdmsg_state_hold(msg->state); */
504 atomic_add_int(&blk_active, 1);
505 atomic_add_int(&iost->count, 1);
506 if (msg->any.head.cmd & DMSGF_DELETE)
507 iost->eof = 1;
508 BUF_KERNPROC(bp);
509 dev_dstrategy(dp->d_rawdev, bio);
510 }
511 if (reterr) {
512 if (msg->any.head.cmd & DMSGF_DELETE) {
513 if (iost && iost->count == 0) {
514 kfree(iost, M_DEVBUF);
515 msg->state->any.any = NULL;
516 }
517 kdmsg_msg_reply(msg, error);
518 } else {
519 kdmsg_msg_result(msg, error);
520 }
521 }
522 }
523
524 static
525 void
disk_blk_freeblks(struct disk * dp,kdmsg_msg_t * msg)526 disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg)
527 {
528 struct dios_io *iost;
529 struct buf *bp;
530 struct bio *bio;
531 int error = DMSG_ERR_NOSUPP;
532 int reterr = 1;
533
534 /*
535 * Only DMSG_BLK_FREEBLKS commands imply read ops.
536 */
537 iost = msg->state->any.any;
538 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FREEBLKS) {
539 if (iost == NULL) {
540 iost = kmalloc(sizeof(*iost), M_DEVBUF,
541 M_WAITOK | M_ZERO);
542 msg->state->any.any = iost;
543 }
544 reterr = 0;
545 bp = getpbuf(NULL);
546 bio = &bp->b_bio1;
547 bp->b_cmd = BUF_CMD_FREEBLKS;
548 bp->b_bcount = msg->any.blk_freeblks.bytes;
549 bp->b_resid = 0;
550 bio->bio_offset = msg->any.blk_freeblks.offset;
551 bio->bio_caller_info1.ptr = msg->state;
552 bio->bio_done = diskiodone;
553
554 /* kdmsg_state_hold(msg->state); */
555 atomic_add_int(&blk_active, 1);
556 atomic_add_int(&iost->count, 1);
557 if (msg->any.head.cmd & DMSGF_DELETE)
558 iost->eof = 1;
559 BUF_KERNPROC(bp);
560 dev_dstrategy(dp->d_rawdev, bio);
561 }
562 if (reterr) {
563 if (msg->any.head.cmd & DMSGF_DELETE) {
564 if (iost && iost->count == 0) {
565 kfree(iost, M_DEVBUF);
566 msg->state->any.any = NULL;
567 }
568 kdmsg_msg_reply(msg, error);
569 } else {
570 kdmsg_msg_result(msg, error);
571 }
572 }
573 }
574
575 static
576 void
diskiodone(struct bio * bio)577 diskiodone(struct bio *bio)
578 {
579 struct buf *bp = bio->bio_buf;
580 kdmsg_state_t *state = bio->bio_caller_info1.ptr;
581 kdmsg_msg_t *rmsg;
582 struct dios_io *iost = state->any.any;
583 int error;
584 int resid = 0;
585 int bytes;
586 uint32_t cmd;
587 void *data;
588
589 cmd = DMSG_LNK_ERROR;
590 data = NULL;
591 bytes = 0;
592
593 switch(bp->b_cmd) {
594 case BUF_CMD_READ:
595 cmd = DMSG_LNK_ERROR;
596 data = bp->b_data;
597 bytes = bp->b_bcount;
598 /* fall through */
599 case BUF_CMD_WRITE:
600 if (bp->b_flags & B_ERROR) {
601 error = bp->b_error;
602 } else {
603 error = 0;
604 resid = bp->b_resid;
605 }
606 kdmsg_free_aux_data(&iost->data);
607 break;
608 case BUF_CMD_FLUSH:
609 case BUF_CMD_FREEBLKS:
610 if (bp->b_flags & B_ERROR)
611 error = bp->b_error;
612 else
613 error = 0;
614 break;
615 default:
616 panic("diskiodone: Unknown bio cmd = %d\n",
617 bio->bio_buf->b_cmd);
618 error = 0; /* avoid compiler warning */
619 break; /* NOT REACHED */
620 }
621
622 /*
623 * Convert error to DMSG_ERR_* code.
624 */
625 if (error)
626 error = DMSG_ERR_IO;
627
628 /*
629 * Convert LNK_ERROR or BLK_ERROR if non-zero resid. READS will
630 * have already converted cmd to BLK_ERROR and set up data to return.
631 */
632 if (resid && cmd == DMSG_LNK_ERROR)
633 cmd = DMSG_BLK_ERROR;
634 /* XXX txcmd is delayed so this won't work for streaming */
635 if ((state->txcmd & DMSGF_CREATE) == 0) /* assume serialized */
636 cmd |= DMSGF_CREATE;
637 if (iost->eof) {
638 if (atomic_fetchadd_int(&iost->count, -1) == 1)
639 cmd |= DMSGF_DELETE;
640 } else {
641 atomic_add_int(&iost->count, -1);
642 }
643 atomic_add_int(&blk_active, -1);
644 cmd |= DMSGF_REPLY;
645
646 /*
647 * Allocate a basic or extended reply. Be careful not to populate
648 * extended header fields unless we allocated an extended reply.
649 */
650 rmsg = kdmsg_msg_alloc(state, cmd, NULL, 0);
651 if (data) {
652 rmsg->aux_data = kmalloc(bytes, state->iocom->mmsg, M_INTWAIT);
653 rmsg->aux_size = bytes;
654 rmsg->flags |= KDMSG_FLAG_AUXALLOC;
655 bcopy(data, rmsg->aux_data, bytes);
656 }
657 rmsg->any.blk_error.head.error = error;
658 if ((cmd & DMSGF_BASECMDMASK) == DMSG_BLK_ERROR)
659 rmsg->any.blk_error.resid = resid;
660 bio->bio_caller_info1.ptr = NULL;
661 /* kdmsg_state_drop(state); */
662 kdmsg_msg_write(rmsg);
663 if (bp->b_flags & B_PAGING) {
664 relpbuf(bp, NULL);
665 } else {
666 bp->b_flags |= B_INVAL | B_AGE;
667 relpbuf(bp, NULL);
668 }
669 }
670