1 /*-
2 * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer,
10 * without modification, immediately at the beginning of the file.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29
30 #include "opt_ada.h"
31
32 #include <sys/param.h>
33
34 #ifdef _KERNEL
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/bio.h>
38 #include <sys/sysctl.h>
39 #include <sys/taskqueue.h>
40 #include <sys/lock.h>
41 #include <sys/mutex.h>
42 #include <sys/conf.h>
43 #include <sys/devicestat.h>
44 #include <sys/eventhandler.h>
45 #include <sys/malloc.h>
46 #include <sys/cons.h>
47 #include <sys/proc.h>
48 #include <sys/reboot.h>
49 #include <geom/geom_disk.h>
50 #endif /* _KERNEL */
51
52 #ifndef _KERNEL
53 #include <stdio.h>
54 #include <string.h>
55 #endif /* _KERNEL */
56
57 #include <cam/cam.h>
58 #include <cam/cam_ccb.h>
59 #include <cam/cam_periph.h>
60 #include <cam/cam_xpt_periph.h>
61 #include <cam/cam_sim.h>
62
63 #include <cam/ata/ata_all.h>
64
65 #include <machine/md_var.h> /* geometry translation */
66
67 #ifdef _KERNEL
68
69 #define ATA_MAX_28BIT_LBA 268435455UL
70
71 typedef enum {
72 ADA_STATE_RAHEAD,
73 ADA_STATE_WCACHE,
74 ADA_STATE_NORMAL
75 } ada_state;
76
77 typedef enum {
78 ADA_FLAG_CAN_48BIT = 0x0002,
79 ADA_FLAG_CAN_FLUSHCACHE = 0x0004,
80 ADA_FLAG_CAN_NCQ = 0x0008,
81 ADA_FLAG_CAN_DMA = 0x0010,
82 ADA_FLAG_NEED_OTAG = 0x0020,
83 ADA_FLAG_WAS_OTAG = 0x0040,
84 ADA_FLAG_CAN_TRIM = 0x0080,
85 ADA_FLAG_OPEN = 0x0100,
86 ADA_FLAG_SCTX_INIT = 0x0200,
87 ADA_FLAG_CAN_CFA = 0x0400,
88 ADA_FLAG_CAN_POWERMGT = 0x0800,
89 ADA_FLAG_CAN_DMA48 = 0x1000,
90 ADA_FLAG_DIRTY = 0x2000
91 } ada_flags;
92
93 typedef enum {
94 ADA_Q_NONE = 0x00,
95 ADA_Q_4K = 0x01,
96 } ada_quirks;
97
98 #define ADA_Q_BIT_STRING \
99 "\020" \
100 "\0014K"
101
102 typedef enum {
103 ADA_CCB_RAHEAD = 0x01,
104 ADA_CCB_WCACHE = 0x02,
105 ADA_CCB_BUFFER_IO = 0x03,
106 ADA_CCB_DUMP = 0x05,
107 ADA_CCB_TRIM = 0x06,
108 ADA_CCB_TYPE_MASK = 0x0F,
109 } ada_ccb_state;
110
111 /* Offsets into our private area for storing information */
112 #define ccb_state ppriv_field0
113 #define ccb_bp ppriv_ptr1
114
115 struct disk_params {
116 u_int8_t heads;
117 u_int8_t secs_per_track;
118 u_int32_t cylinders;
119 u_int32_t secsize; /* Number of bytes/logical sector */
120 u_int64_t sectors; /* Total number sectors */
121 };
122
123 #define TRIM_MAX_BLOCKS 8
124 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
125 struct trim_request {
126 uint8_t data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
127 TAILQ_HEAD(, bio) bps;
128 };
129
130 struct ada_softc {
131 struct bio_queue_head bio_queue;
132 struct bio_queue_head trim_queue;
133 int outstanding_cmds; /* Number of active commands */
134 int refcount; /* Active xpt_action() calls */
135 ada_state state;
136 ada_flags flags;
137 ada_quirks quirks;
138 int sort_io_queue;
139 int trim_max_ranges;
140 int trim_running;
141 int read_ahead;
142 int write_cache;
143 #ifdef ADA_TEST_FAILURE
144 int force_read_error;
145 int force_write_error;
146 int periodic_read_error;
147 int periodic_read_count;
148 #endif
149 struct disk_params params;
150 struct disk *disk;
151 struct task sysctl_task;
152 struct sysctl_ctx_list sysctl_ctx;
153 struct sysctl_oid *sysctl_tree;
154 struct callout sendordered_c;
155 struct trim_request trim_req;
156 };
157
158 struct ada_quirk_entry {
159 struct scsi_inquiry_pattern inq_pat;
160 ada_quirks quirks;
161 };
162
163 static struct ada_quirk_entry ada_quirk_table[] =
164 {
165 {
166 /* Hitachi Advanced Format (4k) drives */
167 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
168 /*quirks*/ADA_Q_4K
169 },
170 {
171 /* Samsung Advanced Format (4k) drives */
172 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
173 /*quirks*/ADA_Q_4K
174 },
175 {
176 /* Samsung Advanced Format (4k) drives */
177 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
178 /*quirks*/ADA_Q_4K
179 },
180 {
181 /* Seagate Barracuda Green Advanced Format (4k) drives */
182 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
183 /*quirks*/ADA_Q_4K
184 },
185 {
186 /* Seagate Barracuda Advanced Format (4k) drives */
187 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
188 /*quirks*/ADA_Q_4K
189 },
190 {
191 /* Seagate Barracuda Advanced Format (4k) drives */
192 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
193 /*quirks*/ADA_Q_4K
194 },
195 {
196 /* Seagate Momentus Advanced Format (4k) drives */
197 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
198 /*quirks*/ADA_Q_4K
199 },
200 {
201 /* Seagate Momentus Advanced Format (4k) drives */
202 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
203 /*quirks*/ADA_Q_4K
204 },
205 {
206 /* Seagate Momentus Advanced Format (4k) drives */
207 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
208 /*quirks*/ADA_Q_4K
209 },
210 {
211 /* Seagate Momentus Advanced Format (4k) drives */
212 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
213 /*quirks*/ADA_Q_4K
214 },
215 {
216 /* Seagate Momentus Advanced Format (4k) drives */
217 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
218 /*quirks*/ADA_Q_4K
219 },
220 {
221 /* Seagate Momentus Advanced Format (4k) drives */
222 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
223 /*quirks*/ADA_Q_4K
224 },
225 {
226 /* Seagate Momentus Advanced Format (4k) drives */
227 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
228 /*quirks*/ADA_Q_4K
229 },
230 {
231 /* Seagate Momentus Thin Advanced Format (4k) drives */
232 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
233 /*quirks*/ADA_Q_4K
234 },
235 {
236 /* WDC Caviar Red Advanced Format (4k) drives */
237 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
238 /*quirks*/ADA_Q_4K
239 },
240 {
241 /* WDC Caviar Green Advanced Format (4k) drives */
242 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
243 /*quirks*/ADA_Q_4K
244 },
245 {
246 /* WDC Caviar Green/Red Advanced Format (4k) drives */
247 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
248 /*quirks*/ADA_Q_4K
249 },
250 {
251 /* WDC Caviar Red Advanced Format (4k) drives */
252 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
253 /*quirks*/ADA_Q_4K
254 },
255 {
256 /* WDC Caviar Black Advanced Format (4k) drives */
257 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????EX*", "*" },
258 /*quirks*/ADA_Q_4K
259 },
260 {
261 /* WDC Caviar Green Advanced Format (4k) drives */
262 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
263 /*quirks*/ADA_Q_4K
264 },
265 {
266 /* WDC Caviar Green Advanced Format (4k) drives */
267 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
268 /*quirks*/ADA_Q_4K
269 },
270 {
271 /* WDC Scorpio Black Advanced Format (4k) drives */
272 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
273 /*quirks*/ADA_Q_4K
274 },
275 {
276 /* WDC Scorpio Black Advanced Format (4k) drives */
277 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
278 /*quirks*/ADA_Q_4K
279 },
280 {
281 /* WDC Scorpio Blue Advanced Format (4k) drives */
282 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
283 /*quirks*/ADA_Q_4K
284 },
285 {
286 /* WDC Scorpio Blue Advanced Format (4k) drives */
287 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
288 /*quirks*/ADA_Q_4K
289 },
290 /* SSDs */
291 {
292 /*
293 * Corsair Force 2 SSDs
294 * 4k optimised & trim only works in 4k requests + 4k aligned
295 */
296 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
297 /*quirks*/ADA_Q_4K
298 },
299 {
300 /*
301 * Corsair Force 3 SSDs
302 * 4k optimised & trim only works in 4k requests + 4k aligned
303 */
304 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
305 /*quirks*/ADA_Q_4K
306 },
307 {
308 /*
309 * Corsair Neutron GTX SSDs
310 * 4k optimised & trim only works in 4k requests + 4k aligned
311 */
312 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
313 /*quirks*/ADA_Q_4K
314 },
315 {
316 /*
317 * Corsair Force GT & GS SSDs
318 * 4k optimised & trim only works in 4k requests + 4k aligned
319 */
320 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
321 /*quirks*/ADA_Q_4K
322 },
323 {
324 /*
325 * Crucial M4 SSDs
326 * 4k optimised & trim only works in 4k requests + 4k aligned
327 */
328 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
329 /*quirks*/ADA_Q_4K
330 },
331 {
332 /*
333 * Crucial RealSSD C300 SSDs
334 * 4k optimised
335 */
336 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
337 "*" }, /*quirks*/ADA_Q_4K
338 },
339 {
340 /*
341 * Intel 320 Series SSDs
342 * 4k optimised & trim only works in 4k requests + 4k aligned
343 */
344 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
345 /*quirks*/ADA_Q_4K
346 },
347 {
348 /*
349 * Intel 330 Series SSDs
350 * 4k optimised & trim only works in 4k requests + 4k aligned
351 */
352 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
353 /*quirks*/ADA_Q_4K
354 },
355 {
356 /*
357 * Intel 510 Series SSDs
358 * 4k optimised & trim only works in 4k requests + 4k aligned
359 */
360 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
361 /*quirks*/ADA_Q_4K
362 },
363 {
364 /*
365 * Intel 520 Series SSDs
366 * 4k optimised & trim only works in 4k requests + 4k aligned
367 */
368 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
369 /*quirks*/ADA_Q_4K
370 },
371 {
372 /*
373 * Intel X25-M Series SSDs
374 * 4k optimised & trim only works in 4k requests + 4k aligned
375 */
376 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
377 /*quirks*/ADA_Q_4K
378 },
379 {
380 /*
381 * Kingston E100 Series SSDs
382 * 4k optimised & trim only works in 4k requests + 4k aligned
383 */
384 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
385 /*quirks*/ADA_Q_4K
386 },
387 {
388 /*
389 * Kingston HyperX 3k SSDs
390 * 4k optimised & trim only works in 4k requests + 4k aligned
391 */
392 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
393 /*quirks*/ADA_Q_4K
394 },
395 {
396 /*
397 * Marvell SSDs (entry taken from OpenSolaris)
398 * 4k optimised & trim only works in 4k requests + 4k aligned
399 */
400 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
401 /*quirks*/ADA_Q_4K
402 },
403 {
404 /*
405 * OCZ Agility 2 SSDs
406 * 4k optimised & trim only works in 4k requests + 4k aligned
407 */
408 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
409 /*quirks*/ADA_Q_4K
410 },
411 {
412 /*
413 * OCZ Agility 3 SSDs
414 * 4k optimised & trim only works in 4k requests + 4k aligned
415 */
416 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
417 /*quirks*/ADA_Q_4K
418 },
419 {
420 /*
421 * OCZ Deneva R Series SSDs
422 * 4k optimised & trim only works in 4k requests + 4k aligned
423 */
424 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
425 /*quirks*/ADA_Q_4K
426 },
427 {
428 /*
429 * OCZ Vertex 2 SSDs (inc pro series)
430 * 4k optimised & trim only works in 4k requests + 4k aligned
431 */
432 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
433 /*quirks*/ADA_Q_4K
434 },
435 {
436 /*
437 * OCZ Vertex 3 SSDs
438 * 4k optimised & trim only works in 4k requests + 4k aligned
439 */
440 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
441 /*quirks*/ADA_Q_4K
442 },
443 {
444 /*
445 * OCZ Vertex 4 SSDs
446 * 4k optimised & trim only works in 4k requests + 4k aligned
447 */
448 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
449 /*quirks*/ADA_Q_4K
450 },
451 {
452 /*
453 * Samsung 830 Series SSDs
454 * 4k optimised
455 */
456 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
457 /*quirks*/ADA_Q_4K
458 },
459 {
460 /*
461 * Samsung 840 SSDs
462 * 4k optimised
463 */
464 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
465 /*quirks*/ADA_Q_4K
466 },
467 {
468 /*
469 * Samsung 843T Series SSDs
470 * 4k optimised
471 */
472 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7WD*", "*" },
473 /*quirks*/ADA_Q_4K
474 },
475 {
476 /*
477 * Samsung 850 SSDs
478 * 4k optimised
479 */
480 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
481 /*quirks*/ADA_Q_4K
482 },
483 {
484 /*
485 * Samsung PM853T Series SSDs
486 * 4k optimised
487 */
488 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7GE*", "*" },
489 /*quirks*/ADA_Q_4K
490 },
491 {
492 /*
493 * SuperTalent TeraDrive CT SSDs
494 * 4k optimised & trim only works in 4k requests + 4k aligned
495 */
496 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
497 /*quirks*/ADA_Q_4K
498 },
499 {
500 /*
501 * XceedIOPS SATA SSDs
502 * 4k optimised
503 */
504 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
505 /*quirks*/ADA_Q_4K
506 },
507 {
508 /* Default */
509 {
510 T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
511 /*vendor*/"*", /*product*/"*", /*revision*/"*"
512 },
513 /*quirks*/0
514 },
515 };
516
517 static disk_strategy_t adastrategy;
518 static dumper_t adadump;
519 static periph_init_t adainit;
520 static void adaasync(void *callback_arg, u_int32_t code,
521 struct cam_path *path, void *arg);
522 static void adasysctlinit(void *context, int pending);
523 static periph_ctor_t adaregister;
524 static periph_dtor_t adacleanup;
525 static periph_start_t adastart;
526 static periph_oninv_t adaoninvalidate;
527 static void adadone(struct cam_periph *periph,
528 union ccb *done_ccb);
529 static int adaerror(union ccb *ccb, u_int32_t cam_flags,
530 u_int32_t sense_flags);
531 static void adagetparams(struct cam_periph *periph,
532 struct ccb_getdev *cgd);
533 static timeout_t adasendorderedtag;
534 static void adashutdown(void *arg, int howto);
535 static void adasuspend(void *arg);
536 static void adaresume(void *arg);
537
538 #ifndef ADA_DEFAULT_LEGACY_ALIASES
539 #define ADA_DEFAULT_LEGACY_ALIASES 1
540 #endif
541
542 #ifndef ADA_DEFAULT_TIMEOUT
543 #define ADA_DEFAULT_TIMEOUT 30 /* Timeout in seconds */
544 #endif
545
546 #ifndef ADA_DEFAULT_RETRY
547 #define ADA_DEFAULT_RETRY 4
548 #endif
549
550 #ifndef ADA_DEFAULT_SEND_ORDERED
551 #define ADA_DEFAULT_SEND_ORDERED 1
552 #endif
553
554 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
555 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN 1
556 #endif
557
558 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
559 #define ADA_DEFAULT_SPINDOWN_SUSPEND 1
560 #endif
561
562 #ifndef ADA_DEFAULT_READ_AHEAD
563 #define ADA_DEFAULT_READ_AHEAD 1
564 #endif
565
566 #ifndef ADA_DEFAULT_WRITE_CACHE
567 #define ADA_DEFAULT_WRITE_CACHE 1
568 #endif
569
570 #define ADA_RA (softc->read_ahead >= 0 ? \
571 softc->read_ahead : ada_read_ahead)
572 #define ADA_WC (softc->write_cache >= 0 ? \
573 softc->write_cache : ada_write_cache)
574 #define ADA_SIO (softc->sort_io_queue >= 0 ? \
575 softc->sort_io_queue : cam_sort_io_queues)
576
577 /*
578 * Most platforms map firmware geometry to actual, but some don't. If
579 * not overridden, default to nothing.
580 */
581 #ifndef ata_disk_firmware_geom_adjust
582 #define ata_disk_firmware_geom_adjust(disk)
583 #endif
584
585 static int ada_retry_count = ADA_DEFAULT_RETRY;
586 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
587 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
588 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
589 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
590 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
591 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
592
593 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
594 "CAM Direct Access Disk driver");
595 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
596 &ada_retry_count, 0, "Normal I/O retry count");
597 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
598 &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
599 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
600 &ada_send_ordered, 0, "Send Ordered Tags");
601 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
602 &ada_spindown_shutdown, 0, "Spin down upon shutdown");
603 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
604 &ada_spindown_suspend, 0, "Spin down upon suspend");
605 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
606 &ada_read_ahead, 0, "Enable disk read-ahead");
607 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
608 &ada_write_cache, 0, "Enable disk write cache");
609
610 /*
611 * ADA_ORDEREDTAG_INTERVAL determines how often, relative
612 * to the default timeout, we check to see whether an ordered
613 * tagged transaction is appropriate to prevent simple tag
614 * starvation. Since we'd like to ensure that there is at least
615 * 1/2 of the timeout length left for a starved transaction to
616 * complete after we've sent an ordered tag, we must poll at least
617 * four times in every timeout period. This takes care of the worst
618 * case where a starved transaction starts during an interval that
619 * meets the requirement "don't send an ordered tag" test so it takes
620 * us two intervals to determine that a tag must be sent.
621 */
622 #ifndef ADA_ORDEREDTAG_INTERVAL
623 #define ADA_ORDEREDTAG_INTERVAL 4
624 #endif
625
626 static struct periph_driver adadriver =
627 {
628 adainit, "ada",
629 TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
630 };
631
632 PERIPHDRIVER_DECLARE(ada, adadriver);
633
634 static int
adaopen(struct disk * dp)635 adaopen(struct disk *dp)
636 {
637 struct cam_periph *periph;
638 struct ada_softc *softc;
639 int error;
640
641 periph = (struct cam_periph *)dp->d_drv1;
642 if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
643 return(ENXIO);
644 }
645
646 cam_periph_lock(periph);
647 if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
648 cam_periph_unlock(periph);
649 cam_periph_release(periph);
650 return (error);
651 }
652
653 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
654 ("adaopen\n"));
655
656 softc = (struct ada_softc *)periph->softc;
657 softc->flags |= ADA_FLAG_OPEN;
658
659 cam_periph_unhold(periph);
660 cam_periph_unlock(periph);
661 return (0);
662 }
663
664 static int
adaclose(struct disk * dp)665 adaclose(struct disk *dp)
666 {
667 struct cam_periph *periph;
668 struct ada_softc *softc;
669 union ccb *ccb;
670 int error;
671
672 periph = (struct cam_periph *)dp->d_drv1;
673 softc = (struct ada_softc *)periph->softc;
674 cam_periph_lock(periph);
675
676 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
677 ("adaclose\n"));
678
679 /* We only sync the cache if the drive is capable of it. */
680 if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
681 (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
682 (periph->flags & CAM_PERIPH_INVALID) == 0 &&
683 cam_periph_hold(periph, PRIBIO) == 0) {
684
685 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
686 cam_fill_ataio(&ccb->ataio,
687 1,
688 adadone,
689 CAM_DIR_NONE,
690 0,
691 NULL,
692 0,
693 ada_default_timeout*1000);
694
695 if (softc->flags & ADA_FLAG_CAN_48BIT)
696 ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
697 else
698 ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
699 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
700 /*sense_flags*/0, softc->disk->d_devstat);
701
702 if (error != 0)
703 xpt_print(periph->path, "Synchronize cache failed\n");
704 else
705 softc->flags &= ~ADA_FLAG_DIRTY;
706 xpt_release_ccb(ccb);
707 cam_periph_unhold(periph);
708 }
709
710 softc->flags &= ~ADA_FLAG_OPEN;
711
712 while (softc->refcount != 0)
713 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
714 cam_periph_unlock(periph);
715 cam_periph_release(periph);
716 return (0);
717 }
718
719 static void
adaschedule(struct cam_periph * periph)720 adaschedule(struct cam_periph *periph)
721 {
722 struct ada_softc *softc = (struct ada_softc *)periph->softc;
723
724 if (softc->state != ADA_STATE_NORMAL)
725 return;
726
727 /* Check if we have more work to do. */
728 if (bioq_first(&softc->bio_queue) ||
729 (!softc->trim_running && bioq_first(&softc->trim_queue))) {
730 xpt_schedule(periph, CAM_PRIORITY_NORMAL);
731 }
732 }
733
734 /*
735 * Actually translate the requested transfer into one the physical driver
736 * can understand. The transfer is described by a buf and will include
737 * only one physical transfer.
738 */
739 static void
adastrategy(struct bio * bp)740 adastrategy(struct bio *bp)
741 {
742 struct cam_periph *periph;
743 struct ada_softc *softc;
744
745 periph = (struct cam_periph *)bp->bio_disk->d_drv1;
746 softc = (struct ada_softc *)periph->softc;
747
748 cam_periph_lock(periph);
749
750 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
751
752 /*
753 * If the device has been made invalid, error out
754 */
755 if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
756 cam_periph_unlock(periph);
757 biofinish(bp, NULL, ENXIO);
758 return;
759 }
760
761 /*
762 * Place it in the queue of disk activities for this disk
763 */
764 if (bp->bio_cmd == BIO_DELETE) {
765 bioq_disksort(&softc->trim_queue, bp);
766 } else {
767 if (ADA_SIO)
768 bioq_disksort(&softc->bio_queue, bp);
769 else
770 bioq_insert_tail(&softc->bio_queue, bp);
771 }
772
773 /*
774 * Schedule ourselves for performing the work.
775 */
776 adaschedule(periph);
777 cam_periph_unlock(periph);
778
779 return;
780 }
781
782 static int
adadump(void * arg,void * virtual,vm_offset_t physical,off_t offset,size_t length)783 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
784 {
785 struct cam_periph *periph;
786 struct ada_softc *softc;
787 u_int secsize;
788 union ccb ccb;
789 struct disk *dp;
790 uint64_t lba;
791 uint16_t count;
792 int error = 0;
793
794 dp = arg;
795 periph = dp->d_drv1;
796 softc = (struct ada_softc *)periph->softc;
797 cam_periph_lock(periph);
798 secsize = softc->params.secsize;
799 lba = offset / secsize;
800 count = length / secsize;
801
802 if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
803 cam_periph_unlock(periph);
804 return (ENXIO);
805 }
806
807 if (length > 0) {
808 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
809 ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
810 cam_fill_ataio(&ccb.ataio,
811 0,
812 adadone,
813 CAM_DIR_OUT,
814 0,
815 (u_int8_t *) virtual,
816 length,
817 ada_default_timeout*1000);
818 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
819 (lba + count >= ATA_MAX_28BIT_LBA ||
820 count >= 256)) {
821 ata_48bit_cmd(&ccb.ataio, ATA_WRITE_DMA48,
822 0, lba, count);
823 } else {
824 ata_28bit_cmd(&ccb.ataio, ATA_WRITE_DMA,
825 0, lba, count);
826 }
827 xpt_polled_action(&ccb);
828
829 error = cam_periph_error(&ccb,
830 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
831 if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
832 cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
833 /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
834 if (error != 0)
835 printf("Aborting dump due to I/O error.\n");
836
837 cam_periph_unlock(periph);
838 return (error);
839 }
840
841 if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
842 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
843
844 ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
845 cam_fill_ataio(&ccb.ataio,
846 0,
847 adadone,
848 CAM_DIR_NONE,
849 0,
850 NULL,
851 0,
852 ada_default_timeout*1000);
853
854 if (softc->flags & ADA_FLAG_CAN_48BIT)
855 ata_48bit_cmd(&ccb.ataio, ATA_FLUSHCACHE48, 0, 0, 0);
856 else
857 ata_28bit_cmd(&ccb.ataio, ATA_FLUSHCACHE, 0, 0, 0);
858 xpt_polled_action(&ccb);
859
860 error = cam_periph_error(&ccb,
861 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
862 if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
863 cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
864 /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
865 if (error != 0)
866 xpt_print(periph->path, "Synchronize cache failed\n");
867 }
868 cam_periph_unlock(periph);
869 return (error);
870 }
871
872 static void
adainit(void)873 adainit(void)
874 {
875 cam_status status;
876
877 /*
878 * Install a global async callback. This callback will
879 * receive async callbacks like "new device found".
880 */
881 status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
882
883 if (status != CAM_REQ_CMP) {
884 printf("ada: Failed to attach master async callback "
885 "due to status 0x%x!\n", status);
886 } else if (ada_send_ordered) {
887
888 /* Register our event handlers */
889 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
890 NULL, EVENTHANDLER_PRI_LAST)) == NULL)
891 printf("adainit: power event registration failed!\n");
892 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
893 NULL, EVENTHANDLER_PRI_LAST)) == NULL)
894 printf("adainit: power event registration failed!\n");
895 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
896 NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
897 printf("adainit: shutdown event registration failed!\n");
898 }
899 }
900
901 /*
902 * Callback from GEOM, called when it has finished cleaning up its
903 * resources.
904 */
905 static void
adadiskgonecb(struct disk * dp)906 adadiskgonecb(struct disk *dp)
907 {
908 struct cam_periph *periph;
909
910 periph = (struct cam_periph *)dp->d_drv1;
911
912 cam_periph_release(periph);
913 }
914
915 static void
adaoninvalidate(struct cam_periph * periph)916 adaoninvalidate(struct cam_periph *periph)
917 {
918 struct ada_softc *softc;
919
920 softc = (struct ada_softc *)periph->softc;
921
922 /*
923 * De-register any async callbacks.
924 */
925 xpt_register_async(0, adaasync, periph, periph->path);
926
927 /*
928 * Return all queued I/O with ENXIO.
929 * XXX Handle any transactions queued to the card
930 * with XPT_ABORT_CCB.
931 */
932 bioq_flush(&softc->bio_queue, NULL, ENXIO);
933 bioq_flush(&softc->trim_queue, NULL, ENXIO);
934
935 disk_gone(softc->disk);
936 }
937
938 static void
adacleanup(struct cam_periph * periph)939 adacleanup(struct cam_periph *periph)
940 {
941 struct ada_softc *softc;
942
943 softc = (struct ada_softc *)periph->softc;
944
945 cam_periph_unlock(periph);
946
947 /*
948 * If we can't free the sysctl tree, oh well...
949 */
950 if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0
951 && sysctl_ctx_free(&softc->sysctl_ctx) != 0) {
952 xpt_print(periph->path, "can't remove sysctl context\n");
953 }
954
955 disk_destroy(softc->disk);
956 callout_drain(&softc->sendordered_c);
957 free(softc, M_DEVBUF);
958 cam_periph_lock(periph);
959 }
960
961 static void
adaasync(void * callback_arg,u_int32_t code,struct cam_path * path,void * arg)962 adaasync(void *callback_arg, u_int32_t code,
963 struct cam_path *path, void *arg)
964 {
965 struct ccb_getdev cgd;
966 struct cam_periph *periph;
967 struct ada_softc *softc;
968
969 periph = (struct cam_periph *)callback_arg;
970 switch (code) {
971 case AC_FOUND_DEVICE:
972 {
973 struct ccb_getdev *cgd;
974 cam_status status;
975
976 cgd = (struct ccb_getdev *)arg;
977 if (cgd == NULL)
978 break;
979
980 if (cgd->protocol != PROTO_ATA)
981 break;
982
983 /*
984 * Allocate a peripheral instance for
985 * this device and start the probe
986 * process.
987 */
988 status = cam_periph_alloc(adaregister, adaoninvalidate,
989 adacleanup, adastart,
990 "ada", CAM_PERIPH_BIO,
991 path, adaasync,
992 AC_FOUND_DEVICE, cgd);
993
994 if (status != CAM_REQ_CMP
995 && status != CAM_REQ_INPROG)
996 printf("adaasync: Unable to attach to new device "
997 "due to status 0x%x\n", status);
998 break;
999 }
1000 case AC_GETDEV_CHANGED:
1001 {
1002 softc = (struct ada_softc *)periph->softc;
1003 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1004 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1005 xpt_action((union ccb *)&cgd);
1006
1007 if ((cgd.ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1008 (cgd.inq_flags & SID_DMA))
1009 softc->flags |= ADA_FLAG_CAN_DMA;
1010 else
1011 softc->flags &= ~ADA_FLAG_CAN_DMA;
1012 if (cgd.ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1013 softc->flags |= ADA_FLAG_CAN_48BIT;
1014 if (cgd.inq_flags & SID_DMA48)
1015 softc->flags |= ADA_FLAG_CAN_DMA48;
1016 else
1017 softc->flags &= ~ADA_FLAG_CAN_DMA48;
1018 } else
1019 softc->flags &= ~(ADA_FLAG_CAN_48BIT |
1020 ADA_FLAG_CAN_DMA48);
1021 if ((cgd.ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1022 (cgd.inq_flags & SID_DMA) && (cgd.inq_flags & SID_CmdQue))
1023 softc->flags |= ADA_FLAG_CAN_NCQ;
1024 else
1025 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1026 if ((cgd.ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1027 (cgd.inq_flags & SID_DMA))
1028 softc->flags |= ADA_FLAG_CAN_TRIM;
1029 else
1030 softc->flags &= ~ADA_FLAG_CAN_TRIM;
1031
1032 cam_periph_async(periph, code, path, arg);
1033 break;
1034 }
1035 case AC_ADVINFO_CHANGED:
1036 {
1037 uintptr_t buftype;
1038
1039 buftype = (uintptr_t)arg;
1040 if (buftype == CDAI_TYPE_PHYS_PATH) {
1041 struct ada_softc *softc;
1042
1043 softc = periph->softc;
1044 disk_attr_changed(softc->disk, "GEOM::physpath",
1045 M_NOWAIT);
1046 }
1047 break;
1048 }
1049 case AC_SENT_BDR:
1050 case AC_BUS_RESET:
1051 {
1052 softc = (struct ada_softc *)periph->softc;
1053 cam_periph_async(periph, code, path, arg);
1054 if (softc->state != ADA_STATE_NORMAL)
1055 break;
1056 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1057 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1058 xpt_action((union ccb *)&cgd);
1059 if (ADA_RA >= 0 &&
1060 cgd.ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1061 softc->state = ADA_STATE_RAHEAD;
1062 else if (ADA_WC >= 0 &&
1063 cgd.ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1064 softc->state = ADA_STATE_WCACHE;
1065 else
1066 break;
1067 if (cam_periph_acquire(periph) != CAM_REQ_CMP)
1068 softc->state = ADA_STATE_NORMAL;
1069 else
1070 xpt_schedule(periph, CAM_PRIORITY_DEV);
1071 }
1072 default:
1073 cam_periph_async(periph, code, path, arg);
1074 break;
1075 }
1076 }
1077
1078 static void
adasysctlinit(void * context,int pending)1079 adasysctlinit(void *context, int pending)
1080 {
1081 struct cam_periph *periph;
1082 struct ada_softc *softc;
1083 char tmpstr[80], tmpstr2[80];
1084
1085 periph = (struct cam_periph *)context;
1086
1087 /* periph was held for us when this task was enqueued */
1088 if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1089 cam_periph_release(periph);
1090 return;
1091 }
1092
1093 softc = (struct ada_softc *)periph->softc;
1094 snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d", periph->unit_number);
1095 snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1096
1097 sysctl_ctx_init(&softc->sysctl_ctx);
1098 softc->flags |= ADA_FLAG_SCTX_INIT;
1099 softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
1100 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1101 CTLFLAG_RD, 0, tmpstr);
1102 if (softc->sysctl_tree == NULL) {
1103 printf("adasysctlinit: unable to allocate sysctl tree\n");
1104 cam_periph_release(periph);
1105 return;
1106 }
1107
1108 SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1109 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1110 &softc->read_ahead, 0, "Enable disk read ahead.");
1111 SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1112 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1113 &softc->write_cache, 0, "Enable disk write cache.");
1114 SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1115 OID_AUTO, "sort_io_queue", CTLFLAG_RW | CTLFLAG_MPSAFE,
1116 &softc->sort_io_queue, 0,
1117 "Sort IO queue to try and optimise disk access patterns");
1118 #ifdef ADA_TEST_FAILURE
1119 /*
1120 * Add a 'door bell' sysctl which allows one to set it from userland
1121 * and cause something bad to happen. For the moment, we only allow
1122 * whacking the next read or write.
1123 */
1124 SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1125 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1126 &softc->force_read_error, 0,
1127 "Force a read error for the next N reads.");
1128 SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1129 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1130 &softc->force_write_error, 0,
1131 "Force a write error for the next N writes.");
1132 SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1133 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1134 &softc->periodic_read_error, 0,
1135 "Force a read error every N reads (don't set too low).");
1136 #endif
1137 cam_periph_release(periph);
1138 }
1139
1140 static int
adagetattr(struct bio * bp)1141 adagetattr(struct bio *bp)
1142 {
1143 int ret;
1144 struct cam_periph *periph;
1145
1146 periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1147 cam_periph_lock(periph);
1148 ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1149 periph->path);
1150 cam_periph_unlock(periph);
1151 if (ret == 0)
1152 bp->bio_completed = bp->bio_length;
1153 return ret;
1154 }
1155
1156 static cam_status
adaregister(struct cam_periph * periph,void * arg)1157 adaregister(struct cam_periph *periph, void *arg)
1158 {
1159 struct ada_softc *softc;
1160 struct ccb_pathinq cpi;
1161 struct ccb_getdev *cgd;
1162 char announce_buf[80];
1163 struct disk_params *dp;
1164 caddr_t match;
1165 u_int maxio;
1166 int quirks;
1167
1168 cgd = (struct ccb_getdev *)arg;
1169 if (cgd == NULL) {
1170 printf("adaregister: no getdev CCB, can't register device\n");
1171 return(CAM_REQ_CMP_ERR);
1172 }
1173
1174 softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1175 M_NOWAIT|M_ZERO);
1176
1177 if (softc == NULL) {
1178 printf("adaregister: Unable to probe new device. "
1179 "Unable to allocate softc\n");
1180 return(CAM_REQ_CMP_ERR);
1181 }
1182
1183 bioq_init(&softc->bio_queue);
1184 bioq_init(&softc->trim_queue);
1185
1186 if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1187 (cgd->inq_flags & SID_DMA))
1188 softc->flags |= ADA_FLAG_CAN_DMA;
1189 if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1190 softc->flags |= ADA_FLAG_CAN_48BIT;
1191 if (cgd->inq_flags & SID_DMA48)
1192 softc->flags |= ADA_FLAG_CAN_DMA48;
1193 }
1194 if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1195 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1196 if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1197 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1198 if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1199 (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1200 softc->flags |= ADA_FLAG_CAN_NCQ;
1201 if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1202 (cgd->inq_flags & SID_DMA)) {
1203 softc->flags |= ADA_FLAG_CAN_TRIM;
1204 softc->trim_max_ranges = TRIM_MAX_RANGES;
1205 if (cgd->ident_data.max_dsm_blocks != 0) {
1206 softc->trim_max_ranges =
1207 min(cgd->ident_data.max_dsm_blocks *
1208 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1209 }
1210 }
1211 if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1212 softc->flags |= ADA_FLAG_CAN_CFA;
1213
1214 periph->softc = softc;
1215
1216 /*
1217 * See if this device has any quirks.
1218 */
1219 match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1220 (caddr_t)ada_quirk_table,
1221 sizeof(ada_quirk_table)/sizeof(*ada_quirk_table),
1222 sizeof(*ada_quirk_table), ata_identify_match);
1223 if (match != NULL)
1224 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1225 else
1226 softc->quirks = ADA_Q_NONE;
1227
1228 bzero(&cpi, sizeof(cpi));
1229 xpt_setup_ccb(&cpi.ccb_h, periph->path, CAM_PRIORITY_NONE);
1230 cpi.ccb_h.func_code = XPT_PATH_INQ;
1231 xpt_action((union ccb *)&cpi);
1232
1233 TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1234
1235 /*
1236 * Register this media as a disk
1237 */
1238 (void)cam_periph_hold(periph, PRIBIO);
1239 cam_periph_unlock(periph);
1240 snprintf(announce_buf, sizeof(announce_buf),
1241 "kern.cam.ada.%d.quirks", periph->unit_number);
1242 quirks = softc->quirks;
1243 TUNABLE_INT_FETCH(announce_buf, &quirks);
1244 softc->quirks = quirks;
1245 softc->read_ahead = -1;
1246 snprintf(announce_buf, sizeof(announce_buf),
1247 "kern.cam.ada.%d.read_ahead", periph->unit_number);
1248 TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1249 softc->write_cache = -1;
1250 snprintf(announce_buf, sizeof(announce_buf),
1251 "kern.cam.ada.%d.write_cache", periph->unit_number);
1252 TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1253 /* Disable queue sorting for non-rotational media by default. */
1254 if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING)
1255 softc->sort_io_queue = 0;
1256 else
1257 softc->sort_io_queue = -1;
1258 adagetparams(periph, cgd);
1259 softc->disk = disk_alloc();
1260 softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
1261 softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1262 periph->unit_number, softc->params.secsize,
1263 DEVSTAT_ALL_SUPPORTED,
1264 DEVSTAT_TYPE_DIRECT |
1265 XPORT_DEVSTAT_TYPE(cpi.transport),
1266 DEVSTAT_PRIORITY_DISK);
1267 softc->disk->d_open = adaopen;
1268 softc->disk->d_close = adaclose;
1269 softc->disk->d_strategy = adastrategy;
1270 softc->disk->d_getattr = adagetattr;
1271 softc->disk->d_dump = adadump;
1272 softc->disk->d_gone = adadiskgonecb;
1273 softc->disk->d_name = "ada";
1274 softc->disk->d_drv1 = periph;
1275 maxio = cpi.maxio; /* Honor max I/O size of SIM */
1276 if (maxio == 0)
1277 maxio = DFLTPHYS; /* traditional default */
1278 else if (maxio > MAXPHYS)
1279 maxio = MAXPHYS; /* for safety */
1280 if (softc->flags & ADA_FLAG_CAN_48BIT)
1281 maxio = min(maxio, 65536 * softc->params.secsize);
1282 else /* 28bit ATA command limit */
1283 maxio = min(maxio, 256 * softc->params.secsize);
1284 softc->disk->d_maxsize = maxio;
1285 softc->disk->d_unit = periph->unit_number;
1286 softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION;
1287 if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1288 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1289 if (softc->flags & ADA_FLAG_CAN_TRIM) {
1290 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1291 softc->disk->d_delmaxsize = softc->params.secsize *
1292 ATA_DSM_RANGE_MAX *
1293 softc->trim_max_ranges;
1294 } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1295 !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1296 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1297 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1298 } else
1299 softc->disk->d_delmaxsize = maxio;
1300 if ((cpi.hba_misc & PIM_UNMAPPED) != 0)
1301 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1302 strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1303 MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1304 strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1305 MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1306 softc->disk->d_hba_vendor = cpi.hba_vendor;
1307 softc->disk->d_hba_device = cpi.hba_device;
1308 softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1309 softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1310
1311 softc->disk->d_sectorsize = softc->params.secsize;
1312 softc->disk->d_mediasize = (off_t)softc->params.sectors *
1313 softc->params.secsize;
1314 if (ata_physical_sector_size(&cgd->ident_data) !=
1315 softc->params.secsize) {
1316 softc->disk->d_stripesize =
1317 ata_physical_sector_size(&cgd->ident_data);
1318 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1319 ata_logical_sector_offset(&cgd->ident_data)) %
1320 softc->disk->d_stripesize;
1321 } else if (softc->quirks & ADA_Q_4K) {
1322 softc->disk->d_stripesize = 4096;
1323 softc->disk->d_stripeoffset = 0;
1324 }
1325 softc->disk->d_fwsectors = softc->params.secs_per_track;
1326 softc->disk->d_fwheads = softc->params.heads;
1327 ata_disk_firmware_geom_adjust(softc->disk);
1328
1329 /*
1330 * Acquire a reference to the periph before we register with GEOM.
1331 * We'll release this reference once GEOM calls us back (via
1332 * adadiskgonecb()) telling us that our provider has been freed.
1333 */
1334 if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
1335 xpt_print(periph->path, "%s: lost periph during "
1336 "registration!\n", __func__);
1337 cam_periph_lock(periph);
1338 return (CAM_REQ_CMP_ERR);
1339 }
1340 disk_create(softc->disk, DISK_VERSION);
1341 cam_periph_lock(periph);
1342 cam_periph_unhold(periph);
1343
1344 dp = &softc->params;
1345 snprintf(announce_buf, sizeof(announce_buf),
1346 "%juMB (%ju %u byte sectors)",
1347 ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1348 (uintmax_t)dp->sectors, dp->secsize);
1349 xpt_announce_periph(periph, announce_buf);
1350 xpt_announce_quirks(periph, softc->quirks, ADA_Q_BIT_STRING);
1351
1352 /*
1353 * Create our sysctl variables, now that we know
1354 * we have successfully attached.
1355 */
1356 if (cam_periph_acquire(periph) == CAM_REQ_CMP)
1357 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1358
1359 /*
1360 * Add async callbacks for bus reset and
1361 * bus device reset calls. I don't bother
1362 * checking if this fails as, in most cases,
1363 * the system will function just fine without
1364 * them and the only alternative would be to
1365 * not attach the device on failure.
1366 */
1367 xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1368 AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1369 adaasync, periph, periph->path);
1370
1371 /*
1372 * Schedule a periodic event to occasionally send an
1373 * ordered tag to a device.
1374 */
1375 callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1376 callout_reset(&softc->sendordered_c,
1377 (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1378 adasendorderedtag, softc);
1379
1380 if (ADA_RA >= 0 &&
1381 cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD) {
1382 softc->state = ADA_STATE_RAHEAD;
1383 } else if (ADA_WC >= 0 &&
1384 cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE) {
1385 softc->state = ADA_STATE_WCACHE;
1386 } else {
1387 softc->state = ADA_STATE_NORMAL;
1388 return(CAM_REQ_CMP);
1389 }
1390 if (cam_periph_acquire(periph) != CAM_REQ_CMP)
1391 softc->state = ADA_STATE_NORMAL;
1392 else
1393 xpt_schedule(periph, CAM_PRIORITY_DEV);
1394 return(CAM_REQ_CMP);
1395 }
1396
1397 static void
ada_dsmtrim(struct ada_softc * softc,struct bio * bp,struct ccb_ataio * ataio)1398 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1399 {
1400 struct trim_request *req = &softc->trim_req;
1401 uint64_t lastlba = (uint64_t)-1;
1402 int c, lastcount = 0, off, ranges = 0;
1403
1404 bzero(req, sizeof(*req));
1405 TAILQ_INIT(&req->bps);
1406 do {
1407 uint64_t lba = bp->bio_pblkno;
1408 int count = bp->bio_bcount / softc->params.secsize;
1409
1410 bioq_remove(&softc->trim_queue, bp);
1411
1412 /* Try to extend the previous range. */
1413 if (lba == lastlba) {
1414 c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1415 lastcount += c;
1416 off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1417 req->data[off + 6] = lastcount & 0xff;
1418 req->data[off + 7] =
1419 (lastcount >> 8) & 0xff;
1420 count -= c;
1421 lba += c;
1422 }
1423
1424 while (count > 0) {
1425 c = min(count, ATA_DSM_RANGE_MAX);
1426 off = ranges * ATA_DSM_RANGE_SIZE;
1427 req->data[off + 0] = lba & 0xff;
1428 req->data[off + 1] = (lba >> 8) & 0xff;
1429 req->data[off + 2] = (lba >> 16) & 0xff;
1430 req->data[off + 3] = (lba >> 24) & 0xff;
1431 req->data[off + 4] = (lba >> 32) & 0xff;
1432 req->data[off + 5] = (lba >> 40) & 0xff;
1433 req->data[off + 6] = c & 0xff;
1434 req->data[off + 7] = (c >> 8) & 0xff;
1435 lba += c;
1436 count -= c;
1437 lastcount = c;
1438 ranges++;
1439 /*
1440 * Its the caller's responsibility to ensure the
1441 * request will fit so we don't need to check for
1442 * overrun here
1443 */
1444 }
1445 lastlba = lba;
1446 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1447 bp = bioq_first(&softc->trim_queue);
1448 if (bp == NULL ||
1449 bp->bio_bcount / softc->params.secsize >
1450 (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX)
1451 break;
1452 } while (1);
1453 cam_fill_ataio(ataio,
1454 ada_retry_count,
1455 adadone,
1456 CAM_DIR_OUT,
1457 0,
1458 req->data,
1459 ((ranges + ATA_DSM_BLK_RANGES - 1) /
1460 ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1461 ada_default_timeout * 1000);
1462 ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
1463 ATA_DSM_TRIM, 0, (ranges + ATA_DSM_BLK_RANGES -
1464 1) / ATA_DSM_BLK_RANGES);
1465 }
1466
1467 static void
ada_cfaerase(struct ada_softc * softc,struct bio * bp,struct ccb_ataio * ataio)1468 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1469 {
1470 struct trim_request *req = &softc->trim_req;
1471 uint64_t lba = bp->bio_pblkno;
1472 uint16_t count = bp->bio_bcount / softc->params.secsize;
1473
1474 bzero(req, sizeof(*req));
1475 TAILQ_INIT(&req->bps);
1476 bioq_remove(&softc->trim_queue, bp);
1477 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1478
1479 cam_fill_ataio(ataio,
1480 ada_retry_count,
1481 adadone,
1482 CAM_DIR_NONE,
1483 0,
1484 NULL,
1485 0,
1486 ada_default_timeout*1000);
1487
1488 if (count >= 256)
1489 count = 0;
1490 ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
1491 }
1492
1493 static void
adastart(struct cam_periph * periph,union ccb * start_ccb)1494 adastart(struct cam_periph *periph, union ccb *start_ccb)
1495 {
1496 struct ada_softc *softc = (struct ada_softc *)periph->softc;
1497 struct ccb_ataio *ataio = &start_ccb->ataio;
1498
1499 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
1500
1501 switch (softc->state) {
1502 case ADA_STATE_NORMAL:
1503 {
1504 struct bio *bp;
1505 u_int8_t tag_code;
1506
1507 /* Run TRIM if not running yet. */
1508 if (!softc->trim_running &&
1509 (bp = bioq_first(&softc->trim_queue)) != 0) {
1510 if (softc->flags & ADA_FLAG_CAN_TRIM) {
1511 ada_dsmtrim(softc, bp, ataio);
1512 } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1513 !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1514 ada_cfaerase(softc, bp, ataio);
1515 } else {
1516 /* This can happen if DMA was disabled. */
1517 bioq_remove(&softc->trim_queue, bp);
1518 biofinish(bp, NULL, EOPNOTSUPP);
1519 xpt_release_ccb(start_ccb);
1520 adaschedule(periph);
1521 return;
1522 }
1523 softc->trim_running = 1;
1524 start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
1525 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
1526 goto out;
1527 }
1528 /* Run regular command. */
1529 bp = bioq_first(&softc->bio_queue);
1530 if (bp == NULL) {
1531 xpt_release_ccb(start_ccb);
1532 break;
1533 }
1534 bioq_remove(&softc->bio_queue, bp);
1535
1536 if ((bp->bio_flags & BIO_ORDERED) != 0
1537 || (softc->flags & ADA_FLAG_NEED_OTAG) != 0) {
1538 softc->flags &= ~ADA_FLAG_NEED_OTAG;
1539 softc->flags |= ADA_FLAG_WAS_OTAG;
1540 tag_code = 0;
1541 } else {
1542 tag_code = 1;
1543 }
1544 switch (bp->bio_cmd) {
1545 case BIO_WRITE:
1546 case BIO_READ:
1547 {
1548 uint64_t lba = bp->bio_pblkno;
1549 uint16_t count = bp->bio_bcount / softc->params.secsize;
1550 void *data_ptr;
1551 int rw_op;
1552
1553 if (bp->bio_cmd == BIO_WRITE) {
1554 softc->flags |= ADA_FLAG_DIRTY;
1555 rw_op = CAM_DIR_OUT;
1556 } else {
1557 rw_op = CAM_DIR_IN;
1558 }
1559
1560 data_ptr = bp->bio_data;
1561 if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
1562 rw_op |= CAM_DATA_BIO;
1563 data_ptr = bp;
1564 }
1565
1566 #ifdef ADA_TEST_FAILURE
1567 int fail = 0;
1568
1569 /*
1570 * Support the failure ioctls. If the command is a
1571 * read, and there are pending forced read errors, or
1572 * if a write and pending write errors, then fail this
1573 * operation with EIO. This is useful for testing
1574 * purposes. Also, support having every Nth read fail.
1575 *
1576 * This is a rather blunt tool.
1577 */
1578 if (bp->bio_cmd == BIO_READ) {
1579 if (softc->force_read_error) {
1580 softc->force_read_error--;
1581 fail = 1;
1582 }
1583 if (softc->periodic_read_error > 0) {
1584 if (++softc->periodic_read_count >=
1585 softc->periodic_read_error) {
1586 softc->periodic_read_count = 0;
1587 fail = 1;
1588 }
1589 }
1590 } else {
1591 if (softc->force_write_error) {
1592 softc->force_write_error--;
1593 fail = 1;
1594 }
1595 }
1596 if (fail) {
1597 biofinish(bp, NULL, EIO);
1598 xpt_release_ccb(start_ccb);
1599 adaschedule(periph);
1600 return;
1601 }
1602 #endif
1603 KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
1604 round_page(bp->bio_bcount + bp->bio_ma_offset) /
1605 PAGE_SIZE == bp->bio_ma_n,
1606 ("Short bio %p", bp));
1607 cam_fill_ataio(ataio,
1608 ada_retry_count,
1609 adadone,
1610 rw_op,
1611 tag_code,
1612 data_ptr,
1613 bp->bio_bcount,
1614 ada_default_timeout*1000);
1615
1616 if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
1617 if (bp->bio_cmd == BIO_READ) {
1618 ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
1619 lba, count);
1620 } else {
1621 ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
1622 lba, count);
1623 }
1624 } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1625 (lba + count >= ATA_MAX_28BIT_LBA ||
1626 count > 256)) {
1627 if (softc->flags & ADA_FLAG_CAN_DMA48) {
1628 if (bp->bio_cmd == BIO_READ) {
1629 ata_48bit_cmd(ataio, ATA_READ_DMA48,
1630 0, lba, count);
1631 } else {
1632 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
1633 0, lba, count);
1634 }
1635 } else {
1636 if (bp->bio_cmd == BIO_READ) {
1637 ata_48bit_cmd(ataio, ATA_READ_MUL48,
1638 0, lba, count);
1639 } else {
1640 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
1641 0, lba, count);
1642 }
1643 }
1644 } else {
1645 if (count == 256)
1646 count = 0;
1647 if (softc->flags & ADA_FLAG_CAN_DMA) {
1648 if (bp->bio_cmd == BIO_READ) {
1649 ata_28bit_cmd(ataio, ATA_READ_DMA,
1650 0, lba, count);
1651 } else {
1652 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
1653 0, lba, count);
1654 }
1655 } else {
1656 if (bp->bio_cmd == BIO_READ) {
1657 ata_28bit_cmd(ataio, ATA_READ_MUL,
1658 0, lba, count);
1659 } else {
1660 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
1661 0, lba, count);
1662 }
1663 }
1664 }
1665 break;
1666 }
1667 case BIO_FLUSH:
1668 cam_fill_ataio(ataio,
1669 1,
1670 adadone,
1671 CAM_DIR_NONE,
1672 0,
1673 NULL,
1674 0,
1675 ada_default_timeout*1000);
1676
1677 if (softc->flags & ADA_FLAG_CAN_48BIT)
1678 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1679 else
1680 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
1681 break;
1682 }
1683 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
1684 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
1685 out:
1686 start_ccb->ccb_h.ccb_bp = bp;
1687 softc->outstanding_cmds++;
1688 softc->refcount++;
1689 cam_periph_unlock(periph);
1690 xpt_action(start_ccb);
1691 cam_periph_lock(periph);
1692 softc->refcount--;
1693
1694 /* May have more work to do, so ensure we stay scheduled */
1695 adaschedule(periph);
1696 break;
1697 }
1698 case ADA_STATE_RAHEAD:
1699 case ADA_STATE_WCACHE:
1700 {
1701 cam_fill_ataio(ataio,
1702 1,
1703 adadone,
1704 CAM_DIR_NONE,
1705 0,
1706 NULL,
1707 0,
1708 ada_default_timeout*1000);
1709
1710 if (softc->state == ADA_STATE_RAHEAD) {
1711 ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
1712 ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
1713 start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
1714 } else {
1715 ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
1716 ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
1717 start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
1718 }
1719 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
1720 xpt_action(start_ccb);
1721 break;
1722 }
1723 }
1724 }
1725
1726 static void
adadone(struct cam_periph * periph,union ccb * done_ccb)1727 adadone(struct cam_periph *periph, union ccb *done_ccb)
1728 {
1729 struct ada_softc *softc;
1730 struct ccb_ataio *ataio;
1731 struct ccb_getdev *cgd;
1732 struct cam_path *path;
1733 int state;
1734
1735 softc = (struct ada_softc *)periph->softc;
1736 ataio = &done_ccb->ataio;
1737 path = done_ccb->ccb_h.path;
1738
1739 CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
1740
1741 state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
1742 switch (state) {
1743 case ADA_CCB_BUFFER_IO:
1744 case ADA_CCB_TRIM:
1745 {
1746 struct bio *bp;
1747 int error;
1748
1749 cam_periph_lock(periph);
1750 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1751 error = adaerror(done_ccb, 0, 0);
1752 if (error == ERESTART) {
1753 /* A retry was scheduled, so just return. */
1754 cam_periph_unlock(periph);
1755 return;
1756 }
1757 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
1758 cam_release_devq(path,
1759 /*relsim_flags*/0,
1760 /*reduction*/0,
1761 /*timeout*/0,
1762 /*getcount_only*/0);
1763 } else {
1764 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
1765 panic("REQ_CMP with QFRZN");
1766 error = 0;
1767 }
1768 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
1769 bp->bio_error = error;
1770 if (error != 0) {
1771 bp->bio_resid = bp->bio_bcount;
1772 bp->bio_flags |= BIO_ERROR;
1773 } else {
1774 if (state == ADA_CCB_TRIM)
1775 bp->bio_resid = 0;
1776 else
1777 bp->bio_resid = ataio->resid;
1778 if (bp->bio_resid > 0)
1779 bp->bio_flags |= BIO_ERROR;
1780 }
1781 softc->outstanding_cmds--;
1782 if (softc->outstanding_cmds == 0)
1783 softc->flags |= ADA_FLAG_WAS_OTAG;
1784 xpt_release_ccb(done_ccb);
1785 if (state == ADA_CCB_TRIM) {
1786 TAILQ_HEAD(, bio) queue;
1787 struct bio *bp1;
1788
1789 TAILQ_INIT(&queue);
1790 TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
1791 /*
1792 * Normally, the xpt_release_ccb() above would make sure
1793 * that when we have more work to do, that work would
1794 * get kicked off. However, we specifically keep
1795 * trim_running set to 0 before the call above to allow
1796 * other I/O to progress when many BIO_DELETE requests
1797 * are pushed down. We set trim_running to 0 and call
1798 * daschedule again so that we don't stall if there are
1799 * no other I/Os pending apart from BIO_DELETEs.
1800 */
1801 softc->trim_running = 0;
1802 adaschedule(periph);
1803 cam_periph_unlock(periph);
1804 while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
1805 TAILQ_REMOVE(&queue, bp1, bio_queue);
1806 bp1->bio_error = error;
1807 if (error != 0) {
1808 bp1->bio_flags |= BIO_ERROR;
1809 bp1->bio_resid = bp1->bio_bcount;
1810 } else
1811 bp1->bio_resid = 0;
1812 biodone(bp1);
1813 }
1814 } else {
1815 cam_periph_unlock(periph);
1816 biodone(bp);
1817 }
1818 return;
1819 }
1820 case ADA_CCB_RAHEAD:
1821 {
1822 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1823 if (adaerror(done_ccb, 0, 0) == ERESTART) {
1824 out:
1825 /* Drop freeze taken due to CAM_DEV_QFREEZE */
1826 cam_release_devq(path, 0, 0, 0, FALSE);
1827 return;
1828 } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
1829 cam_release_devq(path,
1830 /*relsim_flags*/0,
1831 /*reduction*/0,
1832 /*timeout*/0,
1833 /*getcount_only*/0);
1834 }
1835 }
1836
1837 /*
1838 * Since our peripheral may be invalidated by an error
1839 * above or an external event, we must release our CCB
1840 * before releasing the reference on the peripheral.
1841 * The peripheral will only go away once the last reference
1842 * is removed, and we need it around for the CCB release
1843 * operation.
1844 */
1845 cgd = (struct ccb_getdev *)done_ccb;
1846 xpt_setup_ccb(&cgd->ccb_h, path, CAM_PRIORITY_NORMAL);
1847 cgd->ccb_h.func_code = XPT_GDEV_TYPE;
1848 xpt_action((union ccb *)cgd);
1849 if (ADA_WC >= 0 &&
1850 cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE) {
1851 softc->state = ADA_STATE_WCACHE;
1852 xpt_release_ccb(done_ccb);
1853 xpt_schedule(periph, CAM_PRIORITY_DEV);
1854 goto out;
1855 }
1856 softc->state = ADA_STATE_NORMAL;
1857 xpt_release_ccb(done_ccb);
1858 /* Drop freeze taken due to CAM_DEV_QFREEZE */
1859 cam_release_devq(path, 0, 0, 0, FALSE);
1860 adaschedule(periph);
1861 cam_periph_release_locked(periph);
1862 return;
1863 }
1864 case ADA_CCB_WCACHE:
1865 {
1866 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1867 if (adaerror(done_ccb, 0, 0) == ERESTART) {
1868 goto out;
1869 } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
1870 cam_release_devq(path,
1871 /*relsim_flags*/0,
1872 /*reduction*/0,
1873 /*timeout*/0,
1874 /*getcount_only*/0);
1875 }
1876 }
1877
1878 softc->state = ADA_STATE_NORMAL;
1879 /*
1880 * Since our peripheral may be invalidated by an error
1881 * above or an external event, we must release our CCB
1882 * before releasing the reference on the peripheral.
1883 * The peripheral will only go away once the last reference
1884 * is removed, and we need it around for the CCB release
1885 * operation.
1886 */
1887 xpt_release_ccb(done_ccb);
1888 /* Drop freeze taken due to CAM_DEV_QFREEZE */
1889 cam_release_devq(path, 0, 0, 0, FALSE);
1890 adaschedule(periph);
1891 cam_periph_release_locked(periph);
1892 return;
1893 }
1894 case ADA_CCB_DUMP:
1895 /* No-op. We're polling */
1896 return;
1897 default:
1898 break;
1899 }
1900 xpt_release_ccb(done_ccb);
1901 }
1902
1903 static int
adaerror(union ccb * ccb,u_int32_t cam_flags,u_int32_t sense_flags)1904 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
1905 {
1906
1907 return(cam_periph_error(ccb, cam_flags, sense_flags, NULL));
1908 }
1909
1910 static void
adagetparams(struct cam_periph * periph,struct ccb_getdev * cgd)1911 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
1912 {
1913 struct ada_softc *softc = (struct ada_softc *)periph->softc;
1914 struct disk_params *dp = &softc->params;
1915 u_int64_t lbasize48;
1916 u_int32_t lbasize;
1917
1918 dp->secsize = ata_logical_sector_size(&cgd->ident_data);
1919 if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
1920 cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
1921 dp->heads = cgd->ident_data.current_heads;
1922 dp->secs_per_track = cgd->ident_data.current_sectors;
1923 dp->cylinders = cgd->ident_data.cylinders;
1924 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
1925 ((u_int32_t)cgd->ident_data.current_size_2 << 16);
1926 } else {
1927 dp->heads = cgd->ident_data.heads;
1928 dp->secs_per_track = cgd->ident_data.sectors;
1929 dp->cylinders = cgd->ident_data.cylinders;
1930 dp->sectors = cgd->ident_data.cylinders * dp->heads * dp->secs_per_track;
1931 }
1932 lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
1933 ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
1934
1935 /* use the 28bit LBA size if valid or bigger than the CHS mapping */
1936 if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
1937 dp->sectors = lbasize;
1938
1939 /* use the 48bit LBA size if valid */
1940 lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
1941 ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
1942 ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
1943 ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
1944 if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
1945 lbasize48 > ATA_MAX_28BIT_LBA)
1946 dp->sectors = lbasize48;
1947 }
1948
1949 static void
adasendorderedtag(void * arg)1950 adasendorderedtag(void *arg)
1951 {
1952 struct ada_softc *softc = arg;
1953
1954 if (ada_send_ordered) {
1955 if (softc->outstanding_cmds > 0) {
1956 if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
1957 softc->flags |= ADA_FLAG_NEED_OTAG;
1958 softc->flags &= ~ADA_FLAG_WAS_OTAG;
1959 }
1960 }
1961 /* Queue us up again */
1962 callout_reset(&softc->sendordered_c,
1963 (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1964 adasendorderedtag, softc);
1965 }
1966
1967 /*
1968 * Step through all ADA peripheral drivers, and if the device is still open,
1969 * sync the disk cache to physical media.
1970 */
1971 static void
adaflush(void)1972 adaflush(void)
1973 {
1974 struct cam_periph *periph;
1975 struct ada_softc *softc;
1976 union ccb *ccb;
1977 int error;
1978
1979 CAM_PERIPH_FOREACH(periph, &adadriver) {
1980 softc = (struct ada_softc *)periph->softc;
1981 if (SCHEDULER_STOPPED()) {
1982 /* If we paniced with the lock held, do not recurse. */
1983 if (!cam_periph_owned(periph) &&
1984 (softc->flags & ADA_FLAG_OPEN)) {
1985 adadump(softc->disk, NULL, 0, 0, 0);
1986 }
1987 continue;
1988 }
1989 cam_periph_lock(periph);
1990 /*
1991 * We only sync the cache if the drive is still open, and
1992 * if the drive is capable of it..
1993 */
1994 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
1995 (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
1996 cam_periph_unlock(periph);
1997 continue;
1998 }
1999
2000 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
2001 cam_fill_ataio(&ccb->ataio,
2002 0,
2003 adadone,
2004 CAM_DIR_NONE,
2005 0,
2006 NULL,
2007 0,
2008 ada_default_timeout*1000);
2009 if (softc->flags & ADA_FLAG_CAN_48BIT)
2010 ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2011 else
2012 ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
2013
2014 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
2015 /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
2016 softc->disk->d_devstat);
2017 if (error != 0)
2018 xpt_print(periph->path, "Synchronize cache failed\n");
2019 xpt_release_ccb(ccb);
2020 cam_periph_unlock(periph);
2021 }
2022 }
2023
2024 static void
adaspindown(uint8_t cmd,int flags)2025 adaspindown(uint8_t cmd, int flags)
2026 {
2027 struct cam_periph *periph;
2028 struct ada_softc *softc;
2029 union ccb *ccb;
2030 int error;
2031
2032 CAM_PERIPH_FOREACH(periph, &adadriver) {
2033 /* If we paniced with lock held - not recurse here. */
2034 if (cam_periph_owned(periph))
2035 continue;
2036 cam_periph_lock(periph);
2037 softc = (struct ada_softc *)periph->softc;
2038 /*
2039 * We only spin-down the drive if it is capable of it..
2040 */
2041 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
2042 cam_periph_unlock(periph);
2043 continue;
2044 }
2045
2046 if (bootverbose)
2047 xpt_print(periph->path, "spin-down\n");
2048
2049 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
2050 cam_fill_ataio(&ccb->ataio,
2051 0,
2052 adadone,
2053 CAM_DIR_NONE | flags,
2054 0,
2055 NULL,
2056 0,
2057 ada_default_timeout*1000);
2058 ata_28bit_cmd(&ccb->ataio, cmd, 0, 0, 0);
2059
2060 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
2061 /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
2062 softc->disk->d_devstat);
2063 if (error != 0)
2064 xpt_print(periph->path, "Spin-down disk failed\n");
2065 xpt_release_ccb(ccb);
2066 cam_periph_unlock(periph);
2067 }
2068 }
2069
2070 static void
adashutdown(void * arg,int howto)2071 adashutdown(void *arg, int howto)
2072 {
2073
2074 adaflush();
2075 if (ada_spindown_shutdown != 0 &&
2076 (howto & (RB_HALT | RB_POWEROFF)) != 0)
2077 adaspindown(ATA_STANDBY_IMMEDIATE, 0);
2078 }
2079
2080 static void
adasuspend(void * arg)2081 adasuspend(void *arg)
2082 {
2083
2084 adaflush();
2085 if (ada_spindown_suspend != 0)
2086 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
2087 }
2088
2089 static void
adaresume(void * arg)2090 adaresume(void *arg)
2091 {
2092 struct cam_periph *periph;
2093 struct ada_softc *softc;
2094
2095 if (ada_spindown_suspend == 0)
2096 return;
2097
2098 CAM_PERIPH_FOREACH(periph, &adadriver) {
2099 cam_periph_lock(periph);
2100 softc = (struct ada_softc *)periph->softc;
2101 /*
2102 * We only spin-down the drive if it is capable of it..
2103 */
2104 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
2105 cam_periph_unlock(periph);
2106 continue;
2107 }
2108
2109 if (bootverbose)
2110 xpt_print(periph->path, "resume\n");
2111
2112 /*
2113 * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
2114 * sleep request.
2115 */
2116 cam_release_devq(periph->path,
2117 /*relsim_flags*/0,
2118 /*openings*/0,
2119 /*timeout*/0,
2120 /*getcount_only*/0);
2121
2122 cam_periph_unlock(periph);
2123 }
2124 }
2125
2126 #endif /* _KERNEL */
2127