1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
25 * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
26 * LLNL-CODE-403049.
27 */
28
29 #ifndef _ZFS_BLKDEV_H
30 #define _ZFS_BLKDEV_H
31
32 #include <linux/blkdev.h>
33 #include <linux/backing-dev.h>
34 #include <linux/hdreg.h>
35 #include <linux/major.h>
36 #include <linux/msdos_fs.h> /* for SECTOR_* */
37 #include <linux/bio.h>
38 #include <linux/blk-mq.h>
39
40 /*
41 * 6.11 API
42 * Setting the flush flags directly is no longer possible; flush flags are set
43 * on the queue_limits structure and passed to blk_disk_alloc(). In this case
44 * we remove this function entirely.
45 */
46 #if !defined(HAVE_BLK_ALLOC_DISK_2ARG) || \
47 !defined(HAVE_BLKDEV_QUEUE_LIMITS_FEATURES)
48 static inline void
blk_queue_set_write_cache(struct request_queue * q,bool on)49 blk_queue_set_write_cache(struct request_queue *q, bool on)
50 {
51 if (on) {
52 blk_queue_flag_set(QUEUE_FLAG_WC, q);
53 blk_queue_flag_set(QUEUE_FLAG_FUA, q);
54 } else {
55 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
56 blk_queue_flag_clear(QUEUE_FLAG_FUA, q);
57 }
58 }
59 #endif /* !HAVE_BLK_ALLOC_DISK_2ARG || !HAVE_BLKDEV_QUEUE_LIMITS_FEATURES */
60
61 static inline void
blk_queue_set_read_ahead(struct request_queue * q,unsigned long ra_pages)62 blk_queue_set_read_ahead(struct request_queue *q, unsigned long ra_pages)
63 {
64 #if !defined(HAVE_BLK_QUEUE_UPDATE_READAHEAD) && \
65 !defined(HAVE_DISK_UPDATE_READAHEAD)
66 #if defined(HAVE_BLK_QUEUE_BDI_DYNAMIC)
67 q->backing_dev_info->ra_pages = ra_pages;
68 #elif defined(HAVE_BLK_QUEUE_DISK_BDI)
69 q->disk->bdi->ra_pages = ra_pages;
70 #else
71 q->backing_dev_info.ra_pages = ra_pages;
72 #endif
73 #endif
74 }
75
76 #define BIO_BI_SECTOR(bio) (bio)->bi_iter.bi_sector
77 #define BIO_BI_SIZE(bio) (bio)->bi_iter.bi_size
78 #define BIO_BI_IDX(bio) (bio)->bi_iter.bi_idx
79 #define BIO_BI_SKIP(bio) (bio)->bi_iter.bi_bvec_done
80 #define bio_for_each_segment4(bv, bvp, b, i) \
81 bio_for_each_segment((bv), (b), (i))
82 typedef struct bvec_iter bvec_iterator_t;
83
84 static inline void
bio_set_flags_failfast(struct block_device * bdev,int * flags,bool dev,bool transport,bool driver)85 bio_set_flags_failfast(struct block_device *bdev, int *flags, bool dev,
86 bool transport, bool driver)
87 {
88 #ifdef CONFIG_BUG
89 /*
90 * Disable FAILFAST for loopback devices because of the
91 * following incorrect BUG_ON() in loop_make_request().
92 * This support is also disabled for md devices because the
93 * test suite layers md devices on top of loopback devices.
94 * This may be removed when the loopback driver is fixed.
95 *
96 * BUG_ON(!lo || (rw != READ && rw != WRITE));
97 */
98 if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) ||
99 (MAJOR(bdev->bd_dev) == MD_MAJOR))
100 return;
101
102 #ifdef BLOCK_EXT_MAJOR
103 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
104 return;
105 #endif /* BLOCK_EXT_MAJOR */
106 #endif /* CONFIG_BUG */
107
108 if (dev)
109 *flags |= REQ_FAILFAST_DEV;
110 if (transport)
111 *flags |= REQ_FAILFAST_TRANSPORT;
112 if (driver)
113 *flags |= REQ_FAILFAST_DRIVER;
114 }
115
116 /*
117 * Maximum disk label length, it may be undefined for some kernels.
118 */
119 #if !defined(DISK_NAME_LEN)
120 #define DISK_NAME_LEN 32
121 #endif /* DISK_NAME_LEN */
122
123 static inline int
bi_status_to_errno(blk_status_t status)124 bi_status_to_errno(blk_status_t status)
125 {
126 switch (status) {
127 case BLK_STS_OK:
128 return (0);
129 case BLK_STS_NOTSUPP:
130 return (EOPNOTSUPP);
131 case BLK_STS_TIMEOUT:
132 return (ETIMEDOUT);
133 case BLK_STS_NOSPC:
134 return (ENOSPC);
135 case BLK_STS_TRANSPORT:
136 return (ENOLINK);
137 case BLK_STS_TARGET:
138 return (EREMOTEIO);
139 #ifdef HAVE_BLK_STS_RESV_CONFLICT
140 case BLK_STS_RESV_CONFLICT:
141 #else
142 case BLK_STS_NEXUS:
143 #endif
144 return (EBADE);
145 case BLK_STS_MEDIUM:
146 return (ENODATA);
147 case BLK_STS_PROTECTION:
148 return (EILSEQ);
149 case BLK_STS_RESOURCE:
150 return (ENOMEM);
151 case BLK_STS_AGAIN:
152 return (EAGAIN);
153 case BLK_STS_IOERR:
154 return (EIO);
155 default:
156 return (EIO);
157 }
158 }
159
160 static inline blk_status_t
errno_to_bi_status(int error)161 errno_to_bi_status(int error)
162 {
163 switch (error) {
164 case 0:
165 return (BLK_STS_OK);
166 case EOPNOTSUPP:
167 return (BLK_STS_NOTSUPP);
168 case ETIMEDOUT:
169 return (BLK_STS_TIMEOUT);
170 case ENOSPC:
171 return (BLK_STS_NOSPC);
172 case ENOLINK:
173 return (BLK_STS_TRANSPORT);
174 case EREMOTEIO:
175 return (BLK_STS_TARGET);
176 case EBADE:
177 #ifdef HAVE_BLK_STS_RESV_CONFLICT
178 return (BLK_STS_RESV_CONFLICT);
179 #else
180 return (BLK_STS_NEXUS);
181 #endif
182 case ENODATA:
183 return (BLK_STS_MEDIUM);
184 case EILSEQ:
185 return (BLK_STS_PROTECTION);
186 case ENOMEM:
187 return (BLK_STS_RESOURCE);
188 case EAGAIN:
189 return (BLK_STS_AGAIN);
190 case EIO:
191 return (BLK_STS_IOERR);
192 default:
193 return (BLK_STS_IOERR);
194 }
195 }
196
197 /*
198 * 5.15 MACRO,
199 * GD_DEAD
200 *
201 * 2.6.36 - 5.14 MACRO,
202 * GENHD_FL_UP
203 *
204 * Check the disk status and return B_TRUE if alive
205 * otherwise B_FALSE
206 */
207 static inline boolean_t
zfs_check_disk_status(struct block_device * bdev)208 zfs_check_disk_status(struct block_device *bdev)
209 {
210 #if defined(GENHD_FL_UP)
211 return (!!(bdev->bd_disk->flags & GENHD_FL_UP));
212 #elif defined(GD_DEAD)
213 return (!test_bit(GD_DEAD, &bdev->bd_disk->state));
214 #else
215 /*
216 * This is encountered if neither GENHD_FL_UP nor GD_DEAD is available in
217 * the kernel - likely due to an MACRO change that needs to be chased down.
218 */
219 #error "Unsupported kernel: no usable disk status check"
220 #endif
221 }
222
223 /*
224 * 5.17 API change
225 *
226 * GENHD_FL_EXT_DEVT flag removed
227 * GENHD_FL_NO_PART_SCAN renamed GENHD_FL_NO_PART
228 */
229 #ifndef HAVE_GENHD_FL_EXT_DEVT
230 #define GENHD_FL_EXT_DEVT (0)
231 #endif
232 #ifndef HAVE_GENHD_FL_NO_PART
233 #define GENHD_FL_NO_PART (GENHD_FL_NO_PART_SCAN)
234 #endif
235
236 /*
237 * 4.1 API,
238 * 3.10.0 CentOS 7.x API,
239 * blkdev_reread_part()
240 *
241 * For older kernels trigger a re-reading of the partition table by calling
242 * check_disk_change() which calls flush_disk() to invalidate the device.
243 *
244 * For newer kernels (as of 5.10), bdev_check_media_change is used, in favor of
245 * check_disk_change(), with the modification that invalidation is no longer
246 * forced.
247 */
248 #ifdef HAVE_CHECK_DISK_CHANGE
249 #define zfs_check_media_change(bdev) check_disk_change(bdev)
250 #ifdef HAVE_BLKDEV_REREAD_PART
251 #define vdev_bdev_reread_part(bdev) blkdev_reread_part(bdev)
252 #else
253 #define vdev_bdev_reread_part(bdev) check_disk_change(bdev)
254 #endif /* HAVE_BLKDEV_REREAD_PART */
255 #else
256 #ifdef HAVE_BDEV_CHECK_MEDIA_CHANGE
257 static inline int
zfs_check_media_change(struct block_device * bdev)258 zfs_check_media_change(struct block_device *bdev)
259 {
260 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
261 struct gendisk *gd = bdev->bd_disk;
262 const struct block_device_operations *bdo = gd->fops;
263 #endif
264
265 if (!bdev_check_media_change(bdev))
266 return (0);
267
268 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
269 /*
270 * Force revalidation, to mimic the old behavior of
271 * check_disk_change()
272 */
273 if (bdo->revalidate_disk)
274 bdo->revalidate_disk(gd);
275 #endif
276
277 return (0);
278 }
279 #define vdev_bdev_reread_part(bdev) zfs_check_media_change(bdev)
280 #elif defined(HAVE_DISK_CHECK_MEDIA_CHANGE)
281 #define vdev_bdev_reread_part(bdev) disk_check_media_change(bdev->bd_disk)
282 #define zfs_check_media_change(bdev) disk_check_media_change(bdev->bd_disk)
283 #else
284 /*
285 * This is encountered if check_disk_change() and bdev_check_media_change()
286 * are not available in the kernel - likely due to an API change that needs
287 * to be chased down.
288 */
289 #error "Unsupported kernel: no usable disk change check"
290 #endif /* HAVE_BDEV_CHECK_MEDIA_CHANGE */
291 #endif /* HAVE_CHECK_DISK_CHANGE */
292
293 /*
294 * 2.6.27 API change
295 * The function was exported for use, prior to this it existed but the
296 * symbol was not exported.
297 *
298 * 5.11 API change
299 * Changed to take a dev_t argument which is set on success and return a
300 * non-zero error code on failure.
301 */
302 static inline int
vdev_lookup_bdev(const char * path,dev_t * dev)303 vdev_lookup_bdev(const char *path, dev_t *dev)
304 {
305 #if defined(HAVE_DEVT_LOOKUP_BDEV)
306 return (lookup_bdev(path, dev));
307 #elif defined(HAVE_1ARG_LOOKUP_BDEV)
308 struct block_device *bdev = lookup_bdev(path);
309 if (IS_ERR(bdev))
310 return (PTR_ERR(bdev));
311
312 *dev = bdev->bd_dev;
313 bdput(bdev);
314
315 return (0);
316 #else
317 #error "Unsupported kernel"
318 #endif
319 }
320
321 #if defined(HAVE_BLK_MODE_T)
322 #define blk_mode_is_open_write(flag) ((flag) & BLK_OPEN_WRITE)
323 #else
324 #define blk_mode_is_open_write(flag) ((flag) & FMODE_WRITE)
325 #endif
326
327 /*
328 * Kernels without bio_set_op_attrs use bi_rw for the bio flags.
329 */
330 #if !defined(HAVE_BIO_SET_OP_ATTRS)
331 static inline void
bio_set_op_attrs(struct bio * bio,unsigned rw,unsigned flags)332 bio_set_op_attrs(struct bio *bio, unsigned rw, unsigned flags)
333 {
334 bio->bi_opf = rw | flags;
335 }
336 #endif
337
338 /*
339 * bio_set_flush - Set the appropriate flags in a bio to guarantee
340 * data are on non-volatile media on completion.
341 */
342 static inline void
bio_set_flush(struct bio * bio)343 bio_set_flush(struct bio *bio)
344 {
345 bio_set_op_attrs(bio, 0, REQ_PREFLUSH | REQ_OP_WRITE);
346 }
347
348 /*
349 * 4.8 API,
350 * REQ_OP_FLUSH
351 *
352 * in all cases but may have a performance impact for some kernels. It
353 * has the advantage of minimizing kernel specific changes in the zvol code.
354 *
355 */
356 static inline boolean_t
bio_is_flush(struct bio * bio)357 bio_is_flush(struct bio *bio)
358 {
359 return (bio_op(bio) == REQ_OP_FLUSH);
360 }
361
362 /*
363 * 4.8 API,
364 * REQ_FUA flag moved to bio->bi_opf
365 */
366 static inline boolean_t
bio_is_fua(struct bio * bio)367 bio_is_fua(struct bio *bio)
368 {
369 return (bio->bi_opf & REQ_FUA);
370 }
371
372 /*
373 * 4.8 API,
374 * REQ_OP_DISCARD
375 *
376 * In all cases the normal I/O path is used for discards. The only
377 * difference is how the kernel tags individual I/Os as discards.
378 */
379 static inline boolean_t
bio_is_discard(struct bio * bio)380 bio_is_discard(struct bio *bio)
381 {
382 return (bio_op(bio) == REQ_OP_DISCARD);
383 }
384
385 /*
386 * 4.8 API,
387 * REQ_OP_SECURE_ERASE
388 */
389 static inline boolean_t
bio_is_secure_erase(struct bio * bio)390 bio_is_secure_erase(struct bio *bio)
391 {
392 return (bio_op(bio) == REQ_OP_SECURE_ERASE);
393 }
394
395 /*
396 * 2.6.33 API change
397 * Discard granularity and alignment restrictions may now be set. For
398 * older kernels which do not support this it is safe to skip it.
399 */
400 static inline void
blk_queue_discard_granularity(struct request_queue * q,unsigned int dg)401 blk_queue_discard_granularity(struct request_queue *q, unsigned int dg)
402 {
403 q->limits.discard_granularity = dg;
404 }
405
406 /*
407 * 5.19 API,
408 * bdev_max_discard_sectors()
409 *
410 * 2.6.32 API,
411 * blk_queue_discard()
412 */
413 static inline boolean_t
bdev_discard_supported(struct block_device * bdev)414 bdev_discard_supported(struct block_device *bdev)
415 {
416 #if defined(HAVE_BDEV_MAX_DISCARD_SECTORS)
417 return (bdev_max_discard_sectors(bdev) > 0 &&
418 bdev_discard_granularity(bdev) > 0);
419 #elif defined(HAVE_BLK_QUEUE_DISCARD)
420 return (blk_queue_discard(bdev_get_queue(bdev)) > 0 &&
421 bdev_get_queue(bdev)->limits.discard_granularity > 0);
422 #else
423 #error "Unsupported kernel"
424 #endif
425 }
426
427 /*
428 * 5.19 API,
429 * bdev_max_secure_erase_sectors()
430 *
431 * 4.8 API,
432 * blk_queue_secure_erase()
433 */
434 static inline boolean_t
bdev_secure_discard_supported(struct block_device * bdev)435 bdev_secure_discard_supported(struct block_device *bdev)
436 {
437 #if defined(HAVE_BDEV_MAX_SECURE_ERASE_SECTORS)
438 return (!!bdev_max_secure_erase_sectors(bdev));
439 #elif defined(HAVE_BLK_QUEUE_SECURE_ERASE)
440 return (!!blk_queue_secure_erase(bdev_get_queue(bdev)));
441 #else
442 #error "Unsupported kernel"
443 #endif
444 }
445
446 /*
447 * A common holder for vdev_bdev_open() is used to relax the exclusive open
448 * semantics slightly. Internal vdev disk callers may pass VDEV_HOLDER to
449 * allow them to open the device multiple times. Other kernel callers and
450 * user space processes which don't pass this value will get EBUSY. This is
451 * currently required for the correct operation of hot spares.
452 */
453 #define VDEV_HOLDER ((void *)0x2401de7)
454
455 static inline unsigned long
blk_generic_start_io_acct(struct request_queue * q,struct gendisk * disk,int rw,struct bio * bio)456 blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)),
457 struct gendisk *disk __attribute__((unused)),
458 int rw __attribute__((unused)), struct bio *bio)
459 {
460 #if defined(HAVE_BDEV_IO_ACCT_63)
461 return (bdev_start_io_acct(bio->bi_bdev, bio_op(bio),
462 jiffies));
463 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
464 return (bdev_start_io_acct(bio->bi_bdev, bio_sectors(bio),
465 bio_op(bio), jiffies));
466 #elif defined(HAVE_DISK_IO_ACCT)
467 return (disk_start_io_acct(disk, bio_sectors(bio), bio_op(bio)));
468 #elif defined(HAVE_BIO_IO_ACCT)
469 return (bio_start_io_acct(bio));
470 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
471 unsigned long start_time = jiffies;
472 generic_start_io_acct(q, rw, bio_sectors(bio), &disk->part0);
473 return (start_time);
474 #else
475 /* Unsupported */
476 return (0);
477 #endif
478 }
479
480 static inline void
blk_generic_end_io_acct(struct request_queue * q,struct gendisk * disk,int rw,struct bio * bio,unsigned long start_time)481 blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)),
482 struct gendisk *disk __attribute__((unused)),
483 int rw __attribute__((unused)), struct bio *bio, unsigned long start_time)
484 {
485 #if defined(HAVE_BDEV_IO_ACCT_63)
486 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), bio_sectors(bio),
487 start_time);
488 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
489 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time);
490 #elif defined(HAVE_DISK_IO_ACCT)
491 disk_end_io_acct(disk, bio_op(bio), start_time);
492 #elif defined(HAVE_BIO_IO_ACCT)
493 bio_end_io_acct(bio, start_time);
494 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
495 generic_end_io_acct(q, rw, &disk->part0, start_time);
496 #endif
497 }
498
499 #ifndef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
500 static inline struct request_queue *
blk_generic_alloc_queue(make_request_fn make_request,int node_id)501 blk_generic_alloc_queue(make_request_fn make_request, int node_id)
502 {
503 #if defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN)
504 return (blk_alloc_queue(make_request, node_id));
505 #elif defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH)
506 return (blk_alloc_queue_rh(make_request, node_id));
507 #else
508 struct request_queue *q = blk_alloc_queue(GFP_KERNEL);
509 if (q != NULL)
510 blk_queue_make_request(q, make_request);
511
512 return (q);
513 #endif
514 }
515 #endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
516
517 /*
518 * All the io_*() helper functions below can operate on a bio, or a rq, but
519 * not both. The older submit_bio() codepath will pass a bio, and the
520 * newer blk-mq codepath will pass a rq.
521 */
522 static inline int
io_data_dir(struct bio * bio,struct request * rq)523 io_data_dir(struct bio *bio, struct request *rq)
524 {
525 if (rq != NULL) {
526 if (op_is_write(req_op(rq))) {
527 return (WRITE);
528 } else {
529 return (READ);
530 }
531 }
532 return (bio_data_dir(bio));
533 }
534
535 static inline int
io_is_flush(struct bio * bio,struct request * rq)536 io_is_flush(struct bio *bio, struct request *rq)
537 {
538 if (rq != NULL)
539 return (req_op(rq) == REQ_OP_FLUSH);
540 return (bio_is_flush(bio));
541 }
542
543 static inline int
io_is_discard(struct bio * bio,struct request * rq)544 io_is_discard(struct bio *bio, struct request *rq)
545 {
546 if (rq != NULL)
547 return (req_op(rq) == REQ_OP_DISCARD);
548 return (bio_is_discard(bio));
549 }
550
551 static inline int
io_is_secure_erase(struct bio * bio,struct request * rq)552 io_is_secure_erase(struct bio *bio, struct request *rq)
553 {
554 if (rq != NULL)
555 return (req_op(rq) == REQ_OP_SECURE_ERASE);
556 return (bio_is_secure_erase(bio));
557 }
558
559 static inline int
io_is_fua(struct bio * bio,struct request * rq)560 io_is_fua(struct bio *bio, struct request *rq)
561 {
562 if (rq != NULL)
563 return (rq->cmd_flags & REQ_FUA);
564 return (bio_is_fua(bio));
565 }
566
567
568 static inline uint64_t
io_offset(struct bio * bio,struct request * rq)569 io_offset(struct bio *bio, struct request *rq)
570 {
571 if (rq != NULL)
572 return (blk_rq_pos(rq) << 9);
573 return (BIO_BI_SECTOR(bio) << 9);
574 }
575
576 static inline uint64_t
io_size(struct bio * bio,struct request * rq)577 io_size(struct bio *bio, struct request *rq)
578 {
579 if (rq != NULL)
580 return (blk_rq_bytes(rq));
581 return (BIO_BI_SIZE(bio));
582 }
583
584 static inline int
io_has_data(struct bio * bio,struct request * rq)585 io_has_data(struct bio *bio, struct request *rq)
586 {
587 if (rq != NULL)
588 return (bio_has_data(rq->bio));
589 return (bio_has_data(bio));
590 }
591 #endif /* _ZFS_BLKDEV_H */
592