1 /* 2 * Copyright (c) 2016-2018 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 /* 35 * Primary header file 36 * NVME softc and structural definitions 37 */ 38 39 #if defined(__DragonFly__) 40 #include "nvme_dragonfly.h" 41 #else 42 #error "build for OS unknown" 43 #endif 44 #include "nvme_fw.h" 45 #include "nvme_log.h" 46 #include "nvme_ident.h" 47 #include "nvme_ns.h" 48 #include "nvme_chipset.h" 49 #include "nvme_ioctl.h" 50 51 MALLOC_DECLARE(M_NVME); 52 53 /* 54 * Choose some reasonable limit, even if the hardware supports more. 55 */ 56 #define NVME_MAX_QUEUES 1024 57 #define NVME_MAX_NAMESPACES 1024 58 59 struct nvme_queue; 60 struct nvme_softc; 61 struct nvme_softns; 62 63 /* 64 * Device matching array for special attach/detach codings. 65 */ 66 typedef struct { 67 pci_vendor_id_t vendor; 68 pci_product_id_t product; 69 int (*attach)(device_t dev); 70 int (*detach)(device_t dev); 71 char *name; 72 } nvme_device_t; 73 74 /* 75 * Kernel-level request structure. This structure allows the driver to 76 * construct, issue, wait for, and process responses to commands. Each 77 * queue manages its own request bank. 78 * 79 * In order to disconnect the request structure from the hardware queue 80 * mechanism itself, to reduce SMP conflicts and interactions, and allow 81 * command/response processing to block without interfering with queue 82 * operations, this structure embeds a copy of the HW command and response 83 * structures instead of referencing the ones in the actual hardware queues. 84 * These will be copied to/from the actual queue entries by lower-level 85 * chipset code. 86 * 87 * Requests are associated with particular queues, completions can occur on 88 * any queue. Requests associated with the admin queue conveniently include 89 * an additional 4K 'info' block suitable for DMA. 90 */ 91 typedef struct nvme_request { 92 struct nvme_request *next_avail; 93 struct nvme_subqueue *subq; /* which queue is submission on */ 94 struct nvme_comqueue *comq; /* which queue is completion on */ 95 uint32_t state; 96 uint32_t cmd_id; /* reqary[] index */ 97 int waiting; 98 nvme_allcmd_t cmd; /* hw submit structure for entry */ 99 nvme_allres_t res; /* hw completion structure for entry */ 100 nvme_admin_data_t *info; /* DMA data (admin request only) */ 101 bus_addr_t pinfo; /* phys address for PRP */ 102 103 /* 104 * Aux fields to keep track of bio and other data, depending on 105 * the callback. If the callback is NULL the caller will poll for 106 * completion. 107 */ 108 void (*callback)(struct nvme_request *req, struct lock *lk); 109 struct nvme_softns *nsc; 110 struct bio *bio; 111 } nvme_request_t; 112 113 #define NVME_REQ_AVAIL 0 114 #define NVME_REQ_ALLOCATED 1 115 #define NVME_REQ_SUBMITTED 2 116 #define NVME_REQ_COMPLETED 3 117 118 typedef struct nvme_subqueue { 119 /* 120 * Driver stuff 121 */ 122 struct lock lk; /* queue lock controls access */ 123 struct nvme_softc *sc; 124 nvme_request_t *dump_req; /* one request is set aside */ 125 nvme_request_t *first_avail; 126 nvme_request_t *reqary; 127 uint32_t nqe; /* #of queue entries */ 128 uint16_t qid; /* which queue# are we on? */ 129 uint16_t comqid; /* we are tied to this completion qu */ 130 uint32_t subq_doorbell_reg; 131 uint32_t subq_head; /* start of active requests */ 132 uint32_t subq_tail; /* new requests */ 133 uint32_t unsubmitted; /* #unsubmitted requests */ 134 int unused01; 135 int signal_requeue; 136 137 /* 138 * DMA resources 139 */ 140 bus_dmamap_t sque_map; 141 bus_dmamap_t prps_map; 142 nvme_allcmd_t *ksubq; /* kernel-mapped addresses */ 143 uint64_t *kprps; /* enough PRPs per request for */ 144 /* MAXPHYS bytes worth of mappings */ 145 bus_addr_t psubq; /* physical addresses */ 146 bus_addr_t pprps; 147 148 /* 149 * Additional DMA resources for admin queue (A NVME_MAX_ADMIN_BUFFER 150 * sized buffer for each queue entry). 151 */ 152 bus_dmamap_t adm_map; 153 bus_addr_t pdatapgs; 154 nvme_admin_data_t *kdatapgs; 155 } nvme_subqueue_t; 156 157 typedef struct nvme_comqueue { 158 /* 159 * Driver stuff 160 */ 161 struct lock lk; /* queue lock controls access */ 162 struct nvme_softc *sc; 163 uint32_t nqe; /* #of queue entries */ 164 uint16_t phase; /* phase to match (res->tail.status) */ 165 uint16_t qid; /* which queue# are we on? */ 166 uint32_t comq_doorbell_reg; 167 uint32_t comq_head; /* consume responses */ 168 uint32_t comq_tail; 169 170 /* 171 * DMA resources 172 */ 173 bus_dmamap_t cque_map; 174 nvme_allres_t *kcomq; 175 bus_addr_t pcomq; 176 } nvme_comqueue_t; 177 178 typedef struct nvme_softns { 179 struct nvme_softc *sc; 180 nvme_ident_ns_data_t idns; 181 struct bio_queue_head bioq; /* excess BIOs */ 182 struct lock lk; /* mostly for bioq handling */ 183 int state; 184 int unit; 185 uint32_t nsid; 186 uint32_t blksize; 187 struct disk disk; /* disk attachment */ 188 struct devstat stats; 189 cdev_t cdev; /* disk device (cdev) */ 190 } nvme_softns_t; 191 192 #define NVME_NSC_STATE_UNATTACHED 0 193 #define NVME_NSC_STATE_ATTACHED 1 194 195 196 typedef struct nvme_softc { 197 TAILQ_ENTRY(nvme_softc) entry; /* global list */ 198 device_t dev; /* core device */ 199 const nvme_device_t *ad; /* quirk detection etc */ 200 thread_t admintd; 201 uint32_t maxqe; 202 uint64_t cap; 203 uint32_t vers; 204 uint32_t dstrd4; /* doorbell stride */ 205 uint32_t entimo; /* enable timeout in ticks */ 206 uint16_t niosubqs; /* #of I/O submission queues */ 207 uint16_t niocomqs; /* #of I/O completion queues */ 208 uint16_t dumpqno; 209 uint16_t eventqno; 210 uint16_t qmap[SMP_MAXCPU][2]; 211 uint32_t flags; 212 nvme_subqueue_t subqueues[NVME_MAX_QUEUES]; 213 nvme_comqueue_t comqueues[NVME_MAX_QUEUES]; 214 int cputovect[SMP_MAXCPU]; 215 216 /* 217 * bio/disk layer tracking 218 */ 219 ulong opencnt; 220 221 /* 222 * admin queue irq resources 223 * register map resources 224 */ 225 struct resource *regs; 226 struct resource *bar4; 227 bus_space_tag_t iot; 228 bus_space_handle_t ioh; 229 int rid_regs; 230 int rid_bar4; 231 232 int nirqs; 233 int irq_type; 234 struct resource *irq[NVME_MAX_QUEUES]; 235 int rid_irq[NVME_MAX_QUEUES]; 236 void *irq_handle[NVME_MAX_QUEUES]; 237 238 /* 239 * dma tags 240 */ 241 bus_dma_tag_t prps_tag; /* worst-case PRP table(s) per queue */ 242 bus_dma_tag_t sque_tag; /* submission queue */ 243 bus_dma_tag_t cque_tag; /* completion queue */ 244 bus_dma_tag_t adm_tag; /* DMA data buffers for admin cmds */ 245 size_t prp_bytes; 246 size_t cmd_bytes; 247 size_t res_bytes; 248 size_t adm_bytes; 249 250 /* 251 * Admin thread and controller identify data 252 */ 253 uint32_t admin_signal; 254 struct lock admin_lk; 255 struct lock ioctl_lk; 256 int (*admin_func)(struct nvme_softc *); 257 nvme_ident_ctlr_data_t idctlr; 258 nvme_softns_t *nscary[NVME_MAX_NAMESPACES]; 259 int nscmax; 260 } nvme_softc_t; 261 262 #define NVME_SC_ATTACHED 0x00000001 263 #define NVME_SC_UNLOADING 0x00000002 264 265 #define ADMIN_SIG_STOP 0x00000001 266 #define ADMIN_SIG_RUNNING 0x00000002 267 #define ADMIN_SIG_PROBED 0x00000004 268 #define ADMIN_SIG_REQUEUE 0x00000008 269 #define ADMIN_SIG_RUN_MASK (ADMIN_SIG_STOP | ADMIN_SIG_REQUEUE) 270 271 #define NVME_QMAP_RD 0 272 #define NVME_QMAP_WR 1 273 274 typedef struct nvme_status_buf { 275 char buf[64]; 276 } nvme_status_buf_t; 277 278 /* 279 * Prototypes 280 */ 281 const nvme_device_t *nvme_lookup_device(device_t dev); 282 void nvme_os_sleep(int ms); 283 int nvme_os_softsleep(void); 284 void nvme_os_hardsleep(int us); 285 u_int32_t nvme_read(nvme_softc_t *sc, bus_size_t r); 286 u_int64_t nvme_read8(nvme_softc_t *sc, bus_size_t r); 287 void nvme_write(nvme_softc_t *sc, bus_size_t r, u_int32_t v); 288 void nvme_write8(nvme_softc_t *sc, bus_size_t r, u_int64_t v); 289 int nvme_enable(nvme_softc_t *sc, int enable); 290 int nvme_alloc_subqueue(nvme_softc_t *sc, uint16_t qid); 291 int nvme_alloc_comqueue(nvme_softc_t *sc, uint16_t qid); 292 void nvme_free_subqueue(nvme_softc_t *sc, uint16_t qid); 293 void nvme_free_comqueue(nvme_softc_t *sc, uint16_t qid); 294 295 nvme_request_t *nvme_get_admin_request(nvme_softc_t *sc, uint8_t opcode); 296 nvme_request_t *nvme_get_request(nvme_subqueue_t *queue, uint8_t opcode, 297 char *kva, size_t bytes); 298 nvme_request_t *nvme_get_dump_request(nvme_subqueue_t *queue, uint8_t opcode, 299 char *kva, size_t bytes); 300 void nvme_submit_request(nvme_request_t *req); 301 int nvme_wait_request(nvme_request_t *req); 302 int nvme_poll_request(nvme_request_t *req); 303 void nvme_put_request(nvme_request_t *req); 304 void nvme_put_dump_request(nvme_request_t *req); 305 void nvme_poll_completions(nvme_comqueue_t *queue, struct lock *lk); 306 307 int nvme_start_admin_thread(nvme_softc_t *sc); 308 void nvme_stop_admin_thread(nvme_softc_t *sc); 309 310 int nvme_create_subqueue(nvme_softc_t *sc, uint16_t qid); 311 int nvme_create_comqueue(nvme_softc_t *sc, uint16_t qid); 312 int nvme_delete_subqueue(nvme_softc_t *sc, uint16_t qid); 313 int nvme_delete_comqueue(nvme_softc_t *sc, uint16_t qid); 314 int nvme_issue_shutdown(nvme_softc_t *sc, int dopoll); 315 316 void nvme_disk_attach(nvme_softns_t *nsc); 317 void nvme_disk_detach(nvme_softns_t *nsc); 318 void nvme_disk_requeues(nvme_softc_t *sc); 319 int nvme_alloc_disk_unit(void); 320 321 int nvme_getlog_ioctl(nvme_softc_t *sc, nvme_getlog_ioctl_t *ioc); 322 323 void nvme_intr(void *arg); 324 size_t string_cleanup(char *str, int domiddle); 325