1 /*
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2020-2025, Broadcom Inc. All rights reserved.
5 * Support: <fbsd-storage-driver.pdl@broadcom.com>
6 *
7 * Authors: Sumit Saxena <sumit.saxena@broadcom.com>
8 * Chandrakanth Patil <chandrakanth.patil@broadcom.com>
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are
12 * met:
13 *
14 * 1. Redistributions of source code must retain the above copyright notice,
15 * this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright notice,
17 * this list of conditions and the following disclaimer in the documentation and/or other
18 * materials provided with the distribution.
19 * 3. Neither the name of the Broadcom Inc. nor the names of its contributors
20 * may be used to endorse or promote products derived from this software without
21 * specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33 * POSSIBILITY OF SUCH DAMAGE.
34 *
35 * The views and conclusions contained in the software and documentation are
36 * those of the authors and should not be interpreted as representing
37 * official policies,either expressed or implied, of the FreeBSD Project.
38 *
39 * Mail to: Broadcom Inc 1320 Ridder Park Dr, San Jose, CA 95131
40 *
41 * Broadcom Inc. (Broadcom) MPI3MR Adapter FreeBSD
42 */
43
44 #ifndef _MPI3MRVAR_H
45 #define _MPI3MRVAR_H
46
47 #include <sys/cdefs.h>
48 #include <sys/types.h>
49 #include <sys/param.h>
50 #include <sys/systm.h>
51 #include <sys/kernel.h>
52 #include <sys/module.h>
53 #include <sys/bus.h>
54 #include <sys/conf.h>
55 #include <sys/malloc.h>
56 #include <sys/sysctl.h>
57 #include <sys/uio.h>
58 #include <sys/selinfo.h>
59 #include <sys/poll.h>
60
61 #include <sys/lock.h>
62 #include <sys/mutex.h>
63 #include <sys/endian.h>
64 #include <sys/sysent.h>
65 #include <sys/taskqueue.h>
66 #include <sys/smp.h>
67
68 #include <machine/bus.h>
69 #include <machine/resource.h>
70 #include <sys/rman.h>
71
72 #include <dev/pci/pcireg.h>
73 #include <dev/pci/pcivar.h>
74 #include <dev/pci/pci_private.h>
75
76 #include <cam/cam.h>
77 #include <cam/cam_ccb.h>
78 #include <cam/cam_debug.h>
79 #include <cam/cam_sim.h>
80 #include <cam/cam_xpt_sim.h>
81 #include <cam/cam_xpt_periph.h>
82 #include <cam/cam_periph.h>
83 #include <cam/scsi/scsi_all.h>
84 #include <cam/scsi/scsi_message.h>
85
86 #include <cam/scsi/smp_all.h>
87 #include <sys/queue.h>
88 #include <sys/kthread.h>
89 #include "mpi/mpi30_api.h"
90
91 #define MPI3MR_DRIVER_VERSION "8.14.0.2.0"
92 #define MPI3MR_DRIVER_RELDATE "9th Apr 2025"
93
94 #define MPI3MR_DRIVER_NAME "mpi3mr"
95
96 #define MPI3MR_NAME_LENGTH 32
97 #define IOCNAME "%s: "
98
99 #define MPI3MR_DEFAULT_MAX_IO_SIZE (1 * 1024 * 1024)
100
101 #define SAS4116_CHIP_REV_A0 0
102 #define SAS4116_CHIP_REV_B0 1
103
104 #define MPI3MR_MAX_SECTORS 2048
105 #define MPI3MR_MAX_CMDS_LUN 7
106 #define MPI3MR_MAX_CDB_LENGTH 16
107 #define MPI3MR_MAX_LUN 16895
108
109 #define MPI3MR_SATA_QDEPTH 32
110 #define MPI3MR_SAS_QDEPTH 64
111 #define MPI3MR_RAID_QDEPTH 128
112 #define MPI3MR_NVME_QDEPTH 128
113
114 /* Definitions for internal SGL and Chain SGL buffers */
115 #define MPI3MR_4K_PGSZ 4096
116 #define MPI3MR_PAGE_SIZE_4K 4096
117 #define MPI3MR_DEFAULT_SGL_ENTRIES 256
118 #define MPI3MR_MAX_SGL_ENTRIES 2048
119
120 #define MPI3MR_AREQQ_SIZE (2 * MPI3MR_4K_PGSZ)
121 #define MPI3MR_AREPQ_SIZE (4 * MPI3MR_4K_PGSZ)
122 #define MPI3MR_AREQ_FRAME_SZ 128
123 #define MPI3MR_AREP_FRAME_SZ 16
124
125 #define MPI3MR_OPREQQ_SIZE (8 * MPI3MR_4K_PGSZ)
126 #define MPI3MR_OPREPQ_SIZE (4 * MPI3MR_4K_PGSZ)
127
128 /* Operational queue management definitions */
129 #define MPI3MR_OP_REQ_Q_QD 512
130 #define MPI3MR_OP_REP_Q_QD 1024
131 #define MPI3MR_OP_REP_Q_QD_A0 4096
132
133 #define MPI3MR_THRESHOLD_REPLY_COUNT 100
134
135 #define MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST \
136 (MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | MPI3_SGE_FLAGS_DLAS_SYSTEM | \
137 MPI3_SGE_FLAGS_END_OF_LIST)
138
139 #define MPI3MR_HOSTTAG_INVALID 0xFFFF
140 #define MPI3MR_HOSTTAG_INITCMDS 1
141 #define MPI3MR_HOSTTAG_IOCTLCMDS 2
142 #define MPI3MR_HOSTTAG_PELABORT 3
143 #define MPI3MR_HOSTTAG_PELWAIT 4
144 #define MPI3MR_HOSTTAG_TMS 5
145 #define MPI3MR_HOSTTAG_CFGCMDS 6
146
147 #define MAX_MGMT_ADAPTERS 8
148 #define MPI3MR_WAIT_BEFORE_CTRL_RESET 5
149
150 #define MPI3MR_RESET_REASON_OSTYPE_FREEBSD 0x4
151 #define MPI3MR_RESET_REASON_OSTYPE_SHIFT 28
152 #define MPI3MR_RESET_REASON_IOCNUM_SHIFT 20
153
154 struct mpi3mr_mgmt_info {
155 uint16_t count;
156 struct mpi3mr_softc *sc_ptr[MAX_MGMT_ADAPTERS];
157 int max_index;
158 };
159
160 extern char fmt_os_ver[16];
161
162 #define MPI3MR_OS_VERSION(raw_os_ver, fmt_os_ver) sprintf(raw_os_ver, "%d", __FreeBSD_version); \
163 sprintf(fmt_os_ver, "%c%c.%c%c.%c%c%c",\
164 raw_os_ver[0], raw_os_ver[1], raw_os_ver[2],\
165 raw_os_ver[3], raw_os_ver[4], raw_os_ver[5],\
166 raw_os_ver[6]);
167 #define MPI3MR_NUM_DEVRMCMD 1
168 #define MPI3MR_HOSTTAG_DEVRMCMD_MIN (MPI3MR_HOSTTAG_CFGCMDS + 1)
169 #define MPI3MR_HOSTTAG_DEVRMCMD_MAX (MPI3MR_HOSTTAG_DEVRMCMD_MIN + \
170 MPI3MR_NUM_DEVRMCMD - 1)
171 #define MPI3MR_INTERNALCMDS_RESVD MPI3MR_HOSTTAG_DEVRMCMD_MAX
172
173 #define MPI3MR_NUM_EVTACKCMD 4
174 #define MPI3MR_HOSTTAG_EVTACKCMD_MIN (MPI3MR_HOSTTAG_DEVRMCMD_MAX + 1)
175 #define MPI3MR_HOSTTAG_EVTACKCMD_MAX (MPI3MR_HOSTTAG_EVTACKCMD_MIN + \
176 MPI3MR_NUM_EVTACKCMD - 1)
177
178 /* command/controller interaction timeout definitions in seconds */
179 #define MPI3MR_INTADMCMD_TIMEOUT 60
180 #define MPI3MR_PORTENABLE_TIMEOUT 300
181 #define MPI3MR_ABORTTM_TIMEOUT 60
182 #define MPI3MR_RESETTM_TIMEOUT 60
183 #define MPI3MR_TSUPDATE_INTERVAL 900
184 #define MPI3MR_DEFAULT_SHUTDOWN_TIME 120
185 #define MPI3MR_RAID_ERRREC_RESET_TIMEOUT 180
186 #define MPI3MR_RESET_HOST_IOWAIT_TIMEOUT 5
187 #define MPI3MR_PREPARE_FOR_RESET_TIMEOUT 180
188 #define MPI3MR_RESET_ACK_TIMEOUT 30
189 #define MPI3MR_MUR_TIMEOUT 120
190
191 #define MPI3MR_CMD_NOTUSED 0x8000
192 #define MPI3MR_CMD_COMPLETE 0x0001
193 #define MPI3MR_CMD_PENDING 0x0002
194 #define MPI3MR_CMD_REPLYVALID 0x0004
195 #define MPI3MR_CMD_RESET 0x0008
196
197 #define MPI3MR_NUM_EVTREPLIES 64
198 #define MPI3MR_SENSEBUF_SZ 256
199 #define MPI3MR_SENSEBUF_FACTOR 3
200 #define MPI3MR_CHAINBUF_FACTOR 3
201
202 #define MPT3SAS_HOSTPGSZ_4KEXP 12
203
204 #define MPI3MR_INVALID_DEV_HANDLE 0xFFFF
205
206 /* Controller Reset related definitions */
207 #define MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT 5
208 #define MPI3MR_MAX_SHUTDOWN_RETRY_COUNT 2
209
210 /* ResponseCode values */
211 #define MPI3MR_RI_MASK_RESPCODE (0x000000FF)
212 #define MPI3MR_RSP_TM_COMPLETE 0x00
213 #define MPI3MR_RSP_INVALID_FRAME 0x02
214 #define MPI3MR_RSP_TM_NOT_SUPPORTED 0x04
215 #define MPI3MR_RSP_TM_FAILED 0x05
216 #define MPI3MR_RSP_TM_SUCCEEDED 0x08
217 #define MPI3MR_RSP_TM_INVALID_LUN 0x09
218 #define MPI3MR_RSP_TM_OVERLAPPED_TAG 0x0A
219 #define MPI3MR_RSP_IO_QUEUED_ON_IOC \
220 MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC
221
222 /* Definitions for the controller security status*/
223 #define MPI3MR_CTLR_SECURITY_STATUS_MASK 0x0C
224 #define MPI3MR_CTLR_SECURE_DBG_STATUS_MASK 0x02
225
226 #define MPI3MR_INVALID_DEVICE 0x00
227 #define MPI3MR_CONFIG_SECURE_DEVICE 0x04
228 #define MPI3MR_HARD_SECURE_DEVICE 0x08
229 #define MPI3MR_TAMPERED_DEVICE 0x0C
230
231 #define MPI3MR_DEFAULT_MDTS (128 * 1024)
232 #define MPI3MR_DEFAULT_PGSZEXP (12)
233 #define MPI3MR_MAX_IOCTL_TRANSFER_SIZE (1024 * 1024)
234
235 #define MPI3MR_DEVRMHS_RETRYCOUNT 3
236 #define MPI3MR_PELCMDS_RETRYCOUNT 3
237
238 #define MPI3MR_PERIODIC_DELAY 1 /* 1 second heartbeat/watchdog check */
239
240 #define WRITE_SAME_32 0x0d
241
242 #define MPI3MR_TSUPDATE_INTERVAL 900
243
244 struct completion {
245 unsigned int done;
246 struct mtx lock;
247 };
248
249 typedef union {
250 volatile unsigned int val;
251 unsigned int val_rdonly;
252 } mpi3mr_atomic_t;
253
254 #define mpi3mr_atomic_read(v) atomic_load_acq_int(&(v)->val)
255 #define mpi3mr_atomic_set(v,i) atomic_store_rel_int(&(v)->val, i)
256 #define mpi3mr_atomic_dec(v) atomic_subtract_int(&(v)->val, 1)
257 #define mpi3mr_atomic_inc(v) atomic_add_int(&(v)->val, 1)
258 #define mpi3mr_atomic_add(v, u) atomic_add_int(&(v)->val, u)
259 #define mpi3mr_atomic_sub(v, u) atomic_subtract_int(&(v)->val, u)
260
261 /* IOCTL data transfer sge*/
262 #define MPI3MR_NUM_IOCTL_SGE 256
263 #define MPI3MR_IOCTL_SGE_SIZE (8 * 1024)
264
265 struct dma_memory_desc {
266 U32 size;
267 void *addr;
268 bus_dma_tag_t tag;
269 bus_dmamap_t dmamap;
270 bus_addr_t dma_addr;
271 };
272
273 enum mpi3mr_iocstate {
274 MRIOC_STATE_READY = 1,
275 MRIOC_STATE_RESET,
276 MRIOC_STATE_FAULT,
277 MRIOC_STATE_BECOMING_READY,
278 MRIOC_STATE_RESET_REQUESTED,
279 MRIOC_STATE_UNRECOVERABLE,
280 MRIOC_STATE_COUNT,
281 };
282
283 /* Init type definitions */
284 enum mpi3mr_init_type {
285 MPI3MR_INIT_TYPE_INIT = 0,
286 MPI3MR_INIT_TYPE_RESET,
287 MPI3MR_INIT_TYPE_RESUME,
288 };
289
290 /* Reset reason code definitions*/
291 enum mpi3mr_reset_reason {
292 MPI3MR_RESET_FROM_BRINGUP = 1,
293 MPI3MR_RESET_FROM_FAULT_WATCH = 2,
294 MPI3MR_RESET_FROM_IOCTL = 3,
295 MPI3MR_RESET_FROM_EH_HOS = 4,
296 MPI3MR_RESET_FROM_TM_TIMEOUT = 5,
297 MPI3MR_RESET_FROM_IOCTL_TIMEOUT = 6,
298 MPI3MR_RESET_FROM_MUR_FAILURE = 7,
299 MPI3MR_RESET_FROM_CTLR_CLEANUP = 8,
300 MPI3MR_RESET_FROM_CIACTIV_FAULT = 9,
301 MPI3MR_RESET_FROM_PE_TIMEOUT = 10,
302 MPI3MR_RESET_FROM_TSU_TIMEOUT = 11,
303 MPI3MR_RESET_FROM_DELREQQ_TIMEOUT = 12,
304 MPI3MR_RESET_FROM_DELREPQ_TIMEOUT = 13,
305 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT = 14,
306 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT = 15,
307 MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT = 16,
308 MPI3MR_RESET_FROM_IOCINIT_TIMEOUT = 17,
309 MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT = 18,
310 MPI3MR_RESET_FROM_EVTACK_TIMEOUT = 19,
311 MPI3MR_RESET_FROM_CIACTVRST_TIMER = 20,
312 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT = 21,
313 MPI3MR_RESET_FROM_PELABORT_TIMEOUT = 22,
314 MPI3MR_RESET_FROM_SYSFS = 23,
315 MPI3MR_RESET_FROM_SYSFS_TIMEOUT = 24,
316 MPI3MR_RESET_FROM_DIAG_BUFFER_POST_TIMEOUT = 25,
317 MPI3MR_RESET_FROM_SCSIIO_TIMEOUT = 26,
318 MPI3MR_RESET_FROM_FIRMWARE = 27,
319 MPI3MR_DEFAULT_RESET_REASON = 28,
320 MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT = 29,
321 MPI3MR_RESET_REASON_COUNT,
322 };
323
324 struct mpi3mr_compimg_ver
325 {
326 U16 build_num;
327 U16 cust_id;
328 U8 ph_minor;
329 U8 ph_major;
330 U8 gen_minor;
331 U8 gen_major;
332 };
333
334 struct mpi3mr_ioc_facts
335 {
336 U32 ioc_capabilities;
337 struct mpi3mr_compimg_ver fw_ver;
338 U32 mpi_version;
339 U16 max_reqs;
340 U16 product_id;
341 U16 op_req_sz;
342 U16 reply_sz;
343 U16 exceptions;
344 U16 max_perids;
345 U16 max_pds;
346 U16 max_sasexpanders;
347 U32 max_data_length;
348 U16 max_sasinitiators;
349 U16 max_enclosures;
350 U16 max_pcieswitches;
351 U16 max_nvme;
352 U16 max_vds;
353 U16 max_hpds;
354 U16 max_advhpds;
355 U16 max_raidpds;
356 U16 min_devhandle;
357 U16 max_devhandle;
358 U16 max_op_req_q;
359 U16 max_op_reply_q;
360 U16 shutdown_timeout;
361 U8 ioc_num;
362 U8 who_init;
363 U16 max_msix_vectors;
364 U8 personality;
365 U8 dma_mask;
366 U8 protocol_flags;
367 U8 sge_mod_mask;
368 U8 sge_mod_value;
369 U8 sge_mod_shift;
370 U8 max_dev_per_tg;
371 U16 max_io_throttle_group;
372 U16 io_throttle_data_length;
373 U16 io_throttle_low;
374 U16 io_throttle_high;
375 };
376
377 struct mpi3mr_op_req_queue {
378 U16 ci;
379 U16 pi;
380 U16 num_reqs;
381 U8 qid;
382 U8 reply_qid;
383 U32 qsz;
384 void *q_base;
385 bus_dma_tag_t q_base_tag;
386 bus_dmamap_t q_base_dmamap;
387 bus_addr_t q_base_phys;
388 struct mtx q_lock;
389 };
390
391 struct mpi3mr_op_reply_queue {
392 U16 ci;
393 U8 ephase;
394 U8 qid;
395 U16 num_replies;
396 U32 qsz;
397 bus_dma_tag_t q_base_tag;
398 bus_dmamap_t q_base_dmamap;
399 void *q_base;
400 bus_addr_t q_base_phys;
401 mpi3mr_atomic_t pend_ios;
402 bool in_use;
403 struct mtx q_lock;
404 };
405
406 struct irq_info {
407 MPI3_REPLY_DESCRIPTORS_UNION *post_queue;
408 bus_dma_tag_t buffer_dmat;
409 struct resource *irq;
410 void *intrhand;
411 int irq_rid;
412 };
413
414 struct mpi3mr_irq_context {
415 struct mpi3mr_softc *sc;
416 U16 msix_index;
417 struct mpi3mr_op_reply_queue *op_reply_q;
418 char name[MPI3MR_NAME_LENGTH];
419 struct irq_info irq_info;
420 };
421
422 MALLOC_DECLARE(M_MPI3MR);
423 SYSCTL_DECL(_hw_mpi3mr);
424
425 typedef struct mpi3mr_drvr_cmd DRVR_CMD;
426 typedef void (*DRVR_CMD_CALLBACK)(struct mpi3mr_softc *mrioc, DRVR_CMD *drvrcmd);
427 struct mpi3mr_drvr_cmd {
428 struct mtx lock;
429 struct completion completion;
430 void *reply;
431 U8 *sensebuf;
432 U8 iou_rc;
433 U16 state;
434 U16 dev_handle;
435 U16 ioc_status;
436 U32 ioc_loginfo;
437 U8 is_waiting;
438 U8 is_senseprst;
439 U8 retry_count;
440 U16 host_tag;
441 DRVR_CMD_CALLBACK callback;
442 };
443
444 struct mpi3mr_cmd;
445 typedef void mpi3mr_evt_callback_t(struct mpi3mr_softc *, uintptr_t,
446 Mpi3EventNotificationReply_t *reply);
447 typedef void mpi3mr_cmd_callback_t(struct mpi3mr_softc *,
448 struct mpi3mr_cmd *cmd);
449
450 #define MPI3MR_IOVEC_COUNT 2
451
452 enum mpi3mr_data_xfer_direction {
453 MPI3MR_READ = 1,
454 MPI3MR_WRITE,
455 };
456
457 enum mpi3mr_cmd_state {
458 MPI3MR_CMD_STATE_FREE = 1,
459 MPI3MR_CMD_STATE_BUSY,
460 MPI3MR_CMD_STATE_IN_QUEUE,
461 MPI3MR_CMD_STATE_IN_TM,
462 };
463
464 enum mpi3mr_target_state {
465 MPI3MR_DEV_CREATED = 1,
466 MPI3MR_DEV_REMOVE_HS_COMPLETED = 2,
467 };
468
469 struct mpi3mr_cmd {
470 TAILQ_ENTRY(mpi3mr_cmd) next;
471 struct mpi3mr_softc *sc;
472 union ccb *ccb;
473 void *data;
474 u_int length;
475 struct mpi3mr_target *targ;
476 u_int data_dir;
477 u_int state;
478 bus_dmamap_t dmamap;
479 struct scsi_sense_data *sense;
480 struct callout callout;
481 bool callout_owner;
482 U16 hosttag;
483 U8 req_qidx;
484 Mpi3SCSIIORequest_t io_request;
485 };
486
487 struct mpi3mr_chain {
488 bus_dmamap_t buf_dmamap;
489 void *buf;
490 bus_addr_t buf_phys;
491 };
492
493 struct mpi3mr_event_handle {
494 TAILQ_ENTRY(mpi3mr_event_handle) eh_list;
495 mpi3mr_evt_callback_t *callback;
496 void *data;
497 uint8_t mask[16];
498 };
499
500 struct mpi3mr_fw_event_work {
501 U16 event;
502 void *event_data;
503 TAILQ_ENTRY(mpi3mr_fw_event_work) ev_link;
504 U8 send_ack;
505 U8 process_event;
506 U32 event_context;
507 U16 event_data_size;
508 };
509
510 /**
511 * struct delayed_dev_rmhs_node - Delayed device removal node
512 *
513 * @list: list head
514 * @handle: Device handle
515 * @iou_rc: IO Unit Control Reason Code
516 */
517 struct delayed_dev_rmhs_node {
518 TAILQ_ENTRY(delayed_dev_rmhs_node) list;
519 U16 handle;
520 U8 iou_rc;
521 };
522
523 /**
524 * struct delayed_evtack_node - Delayed event ack node
525 *
526 * @list: list head
527 * @event: MPI3 event ID
528 * @event_ctx: Event context
529 */
530 struct delayed_evtack_node {
531 TAILQ_ENTRY(delayed_evtack_node) list;
532 U8 event;
533 U32 event_ctx;
534 };
535
536 /* Reset types */
537 enum reset_type {
538 MPI3MR_NO_RESET,
539 MPI3MR_TRIGGER_SOFT_RESET,
540 };
541
542 struct mpi3mr_reset {
543 u_int type;
544 U32 reason;
545 int status;
546 bool ioctl_reset_snapdump;
547 };
548
549 struct mpi3mr_softc {
550 device_t mpi3mr_dev;
551 struct cdev *mpi3mr_cdev;
552 u_int mpi3mr_flags;
553 #define MPI3MR_FLAGS_SHUTDOWN (1 << 0)
554 #define MPI3MR_FLAGS_DIAGRESET (1 << 1)
555 #define MPI3MR_FLAGS_ATTACH_DONE (1 << 2)
556 #define MPI3MR_FLAGS_PORT_ENABLE_DONE (1 << 3)
557 U8 id;
558 int cpu_count;
559 char name[MPI3MR_NAME_LENGTH];
560 char driver_name[MPI3MR_NAME_LENGTH];
561 int bars;
562 bus_addr_t dma_loaddr;
563 bus_addr_t dma_hiaddr;
564 u_int mpi3mr_debug;
565 struct mpi3mr_reset reset;
566 int max_msix_vectors;
567 int msix_count;
568 bool msix_enable;
569 int io_cmds_highwater;
570 int max_chains;
571 uint32_t chain_frame_size;
572 struct sysctl_ctx_list sysctl_ctx;
573 struct sysctl_oid *sysctl_tree;
574 char fw_version[32];
575 struct mpi3mr_chain *chains;
576 struct callout periodic;
577 struct callout device_check_callout;
578
579 struct mpi3mr_cam_softc *cam_sc;
580 struct mpi3mr_cmd **cmd_list;
581 TAILQ_HEAD(, mpi3mr_cmd) cmd_list_head;
582 struct mtx cmd_pool_lock;
583
584 struct resource *mpi3mr_regs_resource;
585 bus_space_handle_t mpi3mr_bhandle;
586 bus_space_tag_t mpi3mr_btag;
587 int mpi3mr_regs_rid;
588
589 bus_dma_tag_t mpi3mr_parent_dmat;
590 bus_dma_tag_t buffer_dmat;
591
592 int num_reqs;
593 int num_replies;
594 int num_chains;
595
596 TAILQ_HEAD(, mpi3mr_event_handle) event_list;
597 struct mpi3mr_event_handle *mpi3mr_log_eh;
598 struct intr_config_hook mpi3mr_ich;
599
600 struct mtx mpi3mr_mtx;
601 struct mtx io_lock;
602 U8 intr_enabled;
603 TAILQ_HEAD(, delayed_dev_rmhs_node) delayed_rmhs_list;
604 TAILQ_HEAD(, delayed_evtack_node) delayed_evtack_cmds_list;
605
606 U16 num_admin_reqs;
607 U32 admin_req_q_sz;
608 U16 admin_req_pi;
609 U16 admin_req_ci;
610 bus_dma_tag_t admin_req_tag;
611 bus_dmamap_t admin_req_dmamap;
612 bus_addr_t admin_req_phys;
613 U8 *admin_req;
614 struct mtx admin_req_lock;
615
616 U16 num_admin_replies;
617 U32 admin_reply_q_sz;
618 U16 admin_reply_ci;
619 U8 admin_reply_ephase;
620 bus_dma_tag_t admin_reply_tag;
621 bus_dmamap_t admin_reply_dmamap;
622 bus_addr_t admin_reply_phys;
623 U8 *admin_reply;
624 struct mtx admin_reply_lock;
625 bool admin_in_use;
626
627 U32 num_reply_bufs;
628 bus_dma_tag_t reply_buf_tag;
629 bus_dmamap_t reply_buf_dmamap;
630 bus_addr_t reply_buf_phys;
631 U8 *reply_buf;
632 bus_addr_t reply_buf_dma_max_address;
633 bus_addr_t reply_buf_dma_min_address;
634
635 U16 reply_free_q_sz;
636 bus_dma_tag_t reply_free_q_tag;
637 bus_dmamap_t reply_free_q_dmamap;
638 bus_addr_t reply_free_q_phys;
639 U64 *reply_free_q;
640 struct mtx reply_free_q_lock;
641 U32 reply_free_q_host_index;
642
643 U32 num_sense_bufs;
644 bus_dma_tag_t sense_buf_tag;
645 bus_dmamap_t sense_buf_dmamap;
646 bus_addr_t sense_buf_phys;
647 U8 *sense_buf;
648
649 U16 sense_buf_q_sz;
650 bus_dma_tag_t sense_buf_q_tag;
651 bus_dmamap_t sense_buf_q_dmamap;
652 bus_addr_t sense_buf_q_phys;
653 U64 *sense_buf_q;
654 struct mtx sense_buf_q_lock;
655 U32 sense_buf_q_host_index;
656
657 void *nvme_encap_prp_list;
658 bus_addr_t nvme_encap_prp_list_dma;
659 bus_dma_tag_t nvme_encap_prp_list_dmatag;
660 bus_dmamap_t nvme_encap_prp_list_dma_dmamap;
661 U32 nvme_encap_prp_sz;
662
663 U32 ready_timeout;
664
665 struct mpi3mr_irq_context *irq_ctx;
666
667 U16 num_queues; /* Number of request/reply queues */
668 struct mpi3mr_op_req_queue *op_req_q;
669 struct mpi3mr_op_reply_queue *op_reply_q;
670 U16 num_hosttag_op_req_q;
671
672 struct mpi3mr_drvr_cmd init_cmds;
673 struct mpi3mr_ioc_facts facts;
674 U16 reply_sz;
675 U16 op_reply_sz;
676
677 U32 event_masks[MPI3_EVENT_NOTIFY_EVENTMASK_WORDS];
678
679 char fwevt_worker_name[MPI3MR_NAME_LENGTH];
680 struct workqueue_struct *fwevt_worker_thread;
681 struct mtx fwevt_lock;
682 struct mtx target_lock;
683
684 U16 max_host_ios;
685 U32 max_sgl_entries;
686 bus_dma_tag_t chain_sgl_list_tag;
687 struct mpi3mr_chain *chain_sgl_list;
688 U16 chain_bitmap_sz;
689 void *chain_bitmap;
690 struct mtx chain_buf_lock;
691 U16 chain_buf_count;
692
693 struct mpi3mr_drvr_cmd ioctl_cmds;
694 struct mpi3mr_drvr_cmd host_tm_cmds;
695 struct mpi3mr_drvr_cmd dev_rmhs_cmds[MPI3MR_NUM_DEVRMCMD];
696 struct mpi3mr_drvr_cmd evtack_cmds[MPI3MR_NUM_EVTACKCMD];
697 struct mpi3mr_drvr_cmd cfg_cmds;
698
699 U16 devrem_bitmap_sz;
700 void *devrem_bitmap;
701
702 U16 dev_handle_bitmap_sz;
703 void *removepend_bitmap;
704
705 U16 evtack_cmds_bitmap_sz;
706 void *evtack_cmds_bitmap;
707
708 U32 ts_update_counter;
709 U8 reset_in_progress;
710 U8 unrecoverable;
711 U8 block_ioctls;
712 U8 in_prep_ciactv_rst;
713 U16 prep_ciactv_rst_counter;
714 struct mtx reset_mutex;
715
716 U8 prepare_for_reset;
717 U16 prepare_for_reset_timeout_counter;
718
719 U16 diagsave_timeout;
720 int logging_level;
721 U16 flush_io_count;
722
723 Mpi3DriverInfoLayout_t driver_info;
724
725 U16 change_count;
726
727 U8 *log_data_buffer;
728 U16 log_data_buffer_index;
729 U16 log_data_entry_size;
730
731 U8 pel_wait_pend;
732 U8 pel_abort_requested;
733 U8 pel_class;
734 U16 pel_locale;
735
736 struct mpi3mr_drvr_cmd pel_cmds;
737 struct mpi3mr_drvr_cmd pel_abort_cmd;
738 U32 newest_seqnum;
739 void *pel_seq_number;
740 bus_addr_t pel_seq_number_dma;
741 bus_dma_tag_t pel_seq_num_dmatag;
742 bus_dmamap_t pel_seq_num_dmamap;
743 U32 pel_seq_number_sz;
744
745 struct selinfo mpi3mr_select;
746 U32 mpi3mr_poll_waiting;
747 U32 mpi3mr_aen_triggered;
748
749 U16 wait_for_port_enable;
750 U16 track_mapping_events;
751 U16 pending_map_events;
752 mpi3mr_atomic_t fw_outstanding;
753 mpi3mr_atomic_t pend_ioctls;
754 struct proc *watchdog_thread;
755 void *watchdog_chan;
756 void *tm_chan;
757 u_int8_t remove_in_progress;
758 u_int8_t watchdog_thread_active;
759 u_int8_t do_timedout_reset;
760 bool allow_ios;
761 bool secure_ctrl;
762 mpi3mr_atomic_t pend_large_data_sz;
763
764 u_int32_t io_throttle_data_length;
765 u_int32_t io_throttle_high;
766 u_int32_t io_throttle_low;
767 u_int16_t num_io_throttle_group;
768 u_int iot_enable;
769 struct mpi3mr_throttle_group_info *throttle_groups;
770
771 struct dma_memory_desc ioctl_sge[MPI3MR_NUM_IOCTL_SGE];
772 struct dma_memory_desc ioctl_chain_sge;
773 struct dma_memory_desc ioctl_resp_sge;
774 bool ioctl_sges_allocated;
775 struct proc *timestamp_thread_proc;
776 void *timestamp_chan;
777 u_int8_t timestamp_thread_active;
778 U32 ts_update_interval;
779 };
780
781 static __inline uint64_t
mpi3mr_regread64(struct mpi3mr_softc * sc,uint32_t offset)782 mpi3mr_regread64(struct mpi3mr_softc *sc, uint32_t offset)
783 {
784 return bus_space_read_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset);
785 }
786
787 static __inline void
mpi3mr_regwrite64(struct mpi3mr_softc * sc,uint32_t offset,uint64_t val)788 mpi3mr_regwrite64(struct mpi3mr_softc *sc, uint32_t offset, uint64_t val)
789 {
790 bus_space_write_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val);
791 }
792
793 static __inline uint32_t
mpi3mr_regread(struct mpi3mr_softc * sc,uint32_t offset)794 mpi3mr_regread(struct mpi3mr_softc *sc, uint32_t offset)
795 {
796 return bus_space_read_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset);
797 }
798
799 static __inline void
mpi3mr_regwrite(struct mpi3mr_softc * sc,uint32_t offset,uint32_t val)800 mpi3mr_regwrite(struct mpi3mr_softc *sc, uint32_t offset, uint32_t val)
801 {
802 bus_space_write_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val);
803 }
804
805 #define MPI3MR_INFO (1 << 0) /* Basic info */
806 #define MPI3MR_FAULT (1 << 1) /* Hardware faults */
807 #define MPI3MR_EVENT (1 << 2) /* Event data from the controller */
808 #define MPI3MR_LOG (1 << 3) /* Log data from the controller */
809 #define MPI3MR_RECOVERY (1 << 4) /* Command error recovery tracing */
810 #define MPI3MR_ERROR (1 << 5) /* Fatal driver/OS APIs failure */
811 #define MPI3MR_XINFO (1 << 6) /* Additional info logs*/
812 #define MPI3MR_TRACE (1 << 7) /* Trace functions */
813 #define MPI3MR_IOT (1 << 8) /* IO throttling related debugs */
814 #define MPI3MR_DEBUG_TM (1 << 9) /* Task management related debugs */
815 #define MPI3MR_DEBUG_IOCTL (1 << 10) /* IOCTL related debugs */
816
817 #define mpi3mr_printf(sc, args...) \
818 device_printf((sc)->mpi3mr_dev, ##args)
819
820 #define mpi3mr_print_field(sc, msg, args...) \
821 printf("\t" msg, ##args)
822
823 #define mpi3mr_vprintf(sc, args...) \
824 do { \
825 if (bootverbose) \
826 mpi3mr_printf(sc, ##args); \
827 } while (0)
828
829 #define mpi3mr_dprint(sc, level, msg, args...) \
830 do { \
831 if ((sc)->mpi3mr_debug & (level)) \
832 device_printf((sc)->mpi3mr_dev, msg, ##args); \
833 } while (0)
834
835 #define MPI3MR_PRINTFIELD_START(sc, tag...) \
836 mpi3mr_printf((sc), ##tag); \
837 mpi3mr_print_field((sc), ":\n")
838 #define MPI3MR_PRINTFIELD_END(sc, tag) \
839 mpi3mr_printf((sc), tag "\n")
840 #define MPI3MR_PRINTFIELD(sc, facts, attr, fmt) \
841 mpi3mr_print_field((sc), #attr ": " #fmt "\n", (facts)->attr)
842
843 #define mpi3mr_kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg) \
844 kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg)
845 #define mpi3mr_kproc_exit(arg) kproc_exit(arg)
846
847 #if defined(CAM_PRIORITY_XPT)
848 #define MPI3MR_PRIORITY_XPT CAM_PRIORITY_XPT
849 #else
850 #define MPI3MR_PRIORITY_XPT 5
851 #endif
852
853 static __inline void
mpi3mr_clear_bit(int b,volatile void * p)854 mpi3mr_clear_bit(int b, volatile void *p)
855 {
856 atomic_clear_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f));
857 }
858
859 static __inline void
mpi3mr_set_bit(int b,volatile void * p)860 mpi3mr_set_bit(int b, volatile void *p)
861 {
862 atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f));
863 }
864
865 static __inline int
mpi3mr_test_bit(int b,volatile void * p)866 mpi3mr_test_bit(int b, volatile void *p)
867 {
868 return ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f));
869 }
870
871 static __inline int
mpi3mr_test_and_set_bit(int b,volatile void * p)872 mpi3mr_test_and_set_bit(int b, volatile void *p)
873 {
874 int ret = ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f));
875
876 atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f));
877 return ret;
878 }
879
880 static __inline int
mpi3mr_find_first_zero_bit(void * p,int bit_count)881 mpi3mr_find_first_zero_bit(void *p, int bit_count)
882 {
883 int i, sz, j=0;
884 U8 *loc;
885
886 sz = bit_count % 8 ? (bit_count / 8 + 1) : (bit_count / 8);
887 loc = malloc(sz, M_MPI3MR, M_NOWAIT | M_ZERO);
888
889 memcpy(loc, p, sz);
890
891 for (i = 0; i < sz; i++) {
892 j = 0;
893 while (j < 8) {
894 if (!((loc[i] >> j) & 0x1))
895 goto out;
896 j++;
897 }
898 }
899 out:
900 free(loc, M_MPI3MR);
901 return (i + j);
902 }
903
904 #define MPI3MR_DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
905
906 void
907 init_completion(struct completion *completion);
908
909 void
910 complete(struct completion *completion);
911
912 void wait_for_completion_timeout(struct completion *completion,
913 U32 timeout);
914 void wait_for_completion_timeout_tm(struct completion *completion,
915 U32 timeout, struct mpi3mr_softc *sc);
916 void mpi3mr_add_sg_single(void *paddr, U8 flags, U32 length,
917 bus_addr_t dma_addr);
918 void mpi3mr_enable_interrupts(struct mpi3mr_softc *sc);
919 void mpi3mr_disable_interrupts(struct mpi3mr_softc *sc);
920 void mpi3mr_memaddr_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
921 int mpi3mr_submit_admin_cmd(struct mpi3mr_softc *mrioc, void *admin_req,
922 U16 admin_req_sz);
923 int mpi3mr_submit_io(struct mpi3mr_softc *mrioc,
924 struct mpi3mr_op_req_queue *op_req_q, U8 *req);
925 int
926 mpi3mr_alloc_interrupts(struct mpi3mr_softc *sc, U16 setup_one);
927
928 void mpi3mr_cleanup_ioc(struct mpi3mr_softc *sc);
929 int mpi3mr_initialize_ioc(struct mpi3mr_softc *sc, U8 reason);
930 void mpi3mr_build_zero_len_sge(void *paddr);
931 int mpi3mr_issue_event_notification(struct mpi3mr_softc *sc);
932 int
933 mpi3mr_register_events(struct mpi3mr_softc *sc);
934 void mpi3mr_process_op_reply_desc(struct mpi3mr_softc *sc,
935 Mpi3DefaultReplyDescriptor_t *reply_desc, U64 *reply_dma);
936 struct mpi3mr_cmd *
937 mpi3mr_get_command(struct mpi3mr_softc *sc);
938 void
939 mpi3mr_release_command(struct mpi3mr_cmd *cmd);
940 int
941 mpi3mr_complete_io_cmd(struct mpi3mr_softc *sc,
942 struct mpi3mr_irq_context *irq_context);
943 int
944 mpi3mr_cam_detach(struct mpi3mr_softc *sc);
945 int
946 mpi3mr_cam_attach(struct mpi3mr_softc *sc);
947 struct mpi3mr_target *
948 mpi3mr_find_target_by_per_id(struct mpi3mr_cam_softc *cam_sc,
949 uint16_t per_id);
950 struct mpi3mr_target *
951 mpi3mr_find_target_by_dev_handle(struct mpi3mr_cam_softc *cam_sc,
952 uint16_t dev_handle);
953 int mpi3mr_create_device(struct mpi3mr_softc *sc,
954 Mpi3DevicePage0_t *dev_pg0);
955 void
956 mpi3mr_unmap_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd);
957 void
958 init_completion(struct completion *completion);
959 void
960 complete(struct completion *completion);
961 void wait_for_completion_timeout(struct completion *completion,
962 U32 timeout);
963 void
964 poll_for_command_completion(struct mpi3mr_softc *sc,
965 struct mpi3mr_drvr_cmd *cmd, U16 wait);
966 int
967 mpi3mr_alloc_requests(struct mpi3mr_softc *sc);
968 void
969 mpi3mr_watchdog(void *arg);
970 int mpi3mr_issue_port_enable(struct mpi3mr_softc *mrioc, U8 async);
971 void
972 mpi3mr_isr(void *privdata);
973 int
974 mpi3mr_alloc_msix_queues(struct mpi3mr_softc *sc);
975 void
976 mpi3mr_destory_mtx(struct mpi3mr_softc *sc);
977 void
978 mpi3mr_free_mem(struct mpi3mr_softc *sc);
979 void
980 mpi3mr_cleanup_interrupts(struct mpi3mr_softc *sc);
981 int mpi3mr_setup_irqs(struct mpi3mr_softc *sc);
982 void mpi3mr_cleanup_event_taskq(struct mpi3mr_softc *sc);
983 void
984 mpi3mr_hexdump(void *buf, int sz, int format);
985 int mpi3mr_soft_reset_handler(struct mpi3mr_softc *sc,
986 U16 reset_reason, bool snapdump);
987 void
988 mpi3mrsas_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc);
989 void
990 mpi3mr_watchdog_thread(void *arg);
991 void mpi3mr_timestamp_thread(void *arg);
992 void mpi3mr_add_device(struct mpi3mr_softc *sc, U16 per_id);
993 int mpi3mr_remove_device(struct mpi3mr_softc *sc, U16 handle);
994 int
995 mpi3mrsas_register_events(struct mpi3mr_softc *sc);
996 int mpi3mr_process_event_ack(struct mpi3mr_softc *sc, U8 event,
997 U32 event_ctx);
998 int mpi3mr_remove_device_from_os(struct mpi3mr_softc *sc, U16 handle);
999 void mpi3mr_remove_device_from_list(struct mpi3mr_softc *sc, struct mpi3mr_target *target,
1000 bool must_delete);
1001 void mpi3mr_update_device(struct mpi3mr_softc *mrioc,
1002 struct mpi3mr_target *tgtdev, Mpi3DevicePage0_t *dev_pg0, bool is_added);
1003 void mpi3mr_app_save_logdata(struct mpi3mr_softc *sc, char *event_data, U16 event_data_size);
1004 void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_softc *sc,
1005 struct mpi3mr_throttle_group_info *tg, U8 divert_value);
1006 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_softc *sc);
1007 void mpi3mr_poll_pend_io_completions(struct mpi3mr_softc *sc);
1008 void int_to_lun(unsigned int lun, U8 *req_lun);
1009 void trigger_reset_from_watchdog(struct mpi3mr_softc *sc, U8 reset_type, U16 reset_reason);
1010 void mpi3mr_alloc_ioctl_dma_memory(struct mpi3mr_softc *sc);
1011 int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_softc *sc);
1012 #endif /*MPI3MR_H_INCLUDED*/
1013