1 /*-
2 * Copyright (C) 2013 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29
30 #include <sys/param.h>
31 #include <sys/kernel.h>
32 #include <sys/systm.h>
33 #include <sys/bus.h>
34 #include <sys/malloc.h>
35 #include <sys/module.h>
36 #include <sys/queue.h>
37 #include <sys/rman.h>
38 #include <vm/vm.h>
39 #include <vm/pmap.h>
40 #include <machine/bus.h>
41 #include <machine/pmap.h>
42 #include <machine/resource.h>
43 #include <dev/pci/pcireg.h>
44 #include <dev/pci/pcivar.h>
45
46 #include "ntb_regs.h"
47 #include "ntb_hw.h"
48
49 /*
50 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that
51 * allows you to connect two systems using a PCI-e link.
52 *
53 * This module contains the hardware abstraction layer for the NTB. It allows
54 * you to send and recieve interrupts, map the memory windows and send and
55 * receive messages in the scratch-pad registers.
56 *
57 * NOTE: Much of the code in this module is shared with Linux. Any patches may
58 * be picked up and redistributed in Linux with a dual GPL/BSD license.
59 */
60
61 #define NTB_CONFIG_BAR 0
62 #define NTB_B2B_BAR_1 1
63 #define NTB_B2B_BAR_2 2
64 #define NTB_MAX_BARS 3
65 #define NTB_MW_TO_BAR(mw) ((mw) + 1)
66
67 #define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS)
68
69 #define NTB_HB_TIMEOUT 1 /* second */
70 #define SOC_LINK_RECOVERY_TIME 500
71
72 #define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev))
73
74 enum ntb_device_type {
75 NTB_XEON,
76 NTB_SOC
77 };
78
79 /* Device features and workarounds */
80 #define HAS_FEATURE(feature) \
81 ((ntb->features & (feature)) != 0)
82
83 #define NTB_BAR_SIZE_4K (1 << 0)
84 #define NTB_REGS_THRU_MW (1 << 1)
85
86 struct ntb_hw_info {
87 uint32_t device_id;
88 const char *desc;
89 enum ntb_device_type type;
90 uint64_t features;
91 };
92
93 struct ntb_pci_bar_info {
94 bus_space_tag_t pci_bus_tag;
95 bus_space_handle_t pci_bus_handle;
96 int pci_resource_id;
97 struct resource *pci_resource;
98 vm_paddr_t pbase;
99 void *vbase;
100 u_long size;
101 };
102
103 struct ntb_int_info {
104 struct resource *res;
105 int rid;
106 void *tag;
107 };
108
109 struct ntb_db_cb {
110 ntb_db_callback callback;
111 unsigned int db_num;
112 void *data;
113 struct ntb_softc *ntb;
114 };
115
116 struct ntb_softc {
117 device_t device;
118 enum ntb_device_type type;
119 uint64_t features;
120
121 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS];
122 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS];
123 uint32_t allocated_interrupts;
124
125 struct callout heartbeat_timer;
126 struct callout lr_timer;
127
128 void *ntb_transport;
129 ntb_event_callback event_cb;
130 struct ntb_db_cb *db_cb;
131
132 struct {
133 uint32_t max_spads;
134 uint32_t max_db_bits;
135 uint32_t msix_cnt;
136 } limits;
137 struct {
138 uint32_t pdb;
139 uint32_t pdb_mask;
140 uint32_t sdb;
141 uint32_t sbar2_xlat;
142 uint32_t sbar4_xlat;
143 uint32_t spad_remote;
144 uint32_t spad_local;
145 uint32_t lnk_cntl;
146 uint32_t lnk_stat;
147 uint32_t spci_cmd;
148 } reg_ofs;
149 uint8_t conn_type;
150 uint8_t dev_type;
151 uint8_t bits_per_vector;
152 uint8_t link_status;
153 uint8_t link_width;
154 uint8_t link_speed;
155 };
156
157 #define ntb_bar_read(SIZE, bar, offset) \
158 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
159 ntb->bar_info[(bar)].pci_bus_handle, (offset))
160 #define ntb_bar_write(SIZE, bar, offset, val) \
161 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
162 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val))
163 #define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset)
164 #define ntb_reg_write(SIZE, offset, val) \
165 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
166 #define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset)
167 #define ntb_mw_write(SIZE, offset, val) \
168 ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val)
169
170 typedef int (*bar_map_strategy)(struct ntb_softc *ntb,
171 struct ntb_pci_bar_info *bar);
172
173 static int ntb_probe(device_t device);
174 static int ntb_attach(device_t device);
175 static int ntb_detach(device_t device);
176 static int ntb_map_pci_bars(struct ntb_softc *ntb);
177 static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
178 struct ntb_pci_bar_info *bar);
179 static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
180 static int map_memory_window_bar(struct ntb_softc *ntb,
181 struct ntb_pci_bar_info *bar);
182 static void ntb_unmap_pci_bar(struct ntb_softc *ntb);
183 static int ntb_setup_interrupts(struct ntb_softc *ntb);
184 static void ntb_teardown_interrupts(struct ntb_softc *ntb);
185 static void handle_soc_irq(void *arg);
186 static void handle_xeon_irq(void *arg);
187 static void handle_xeon_event_irq(void *arg);
188 static void ntb_handle_legacy_interrupt(void *arg);
189 static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors);
190 static void ntb_free_callbacks(struct ntb_softc *ntb);
191 static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
192 static int ntb_initialize_hw(struct ntb_softc *ntb);
193 static int ntb_setup_xeon(struct ntb_softc *ntb);
194 static int ntb_setup_soc(struct ntb_softc *ntb);
195 static void configure_soc_secondary_side_bars(struct ntb_softc *ntb);
196 static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb);
197 static void ntb_handle_heartbeat(void *arg);
198 static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state);
199 static void recover_soc_link(void *arg);
200 static int ntb_check_link_status(struct ntb_softc *ntb);
201 static void save_bar_parameters(struct ntb_pci_bar_info *bar);
202
203 static struct ntb_hw_info pci_ids[] = {
204 { 0x3C0D8086, "Xeon E5/Core i7 Non-Transparent Bridge B2B", NTB_XEON,
205 NTB_REGS_THRU_MW },
206 { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 },
207 { 0x0E0D8086, "Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
208 NTB_REGS_THRU_MW | NTB_BAR_SIZE_4K },
209 { 0x00000000, NULL, NTB_SOC, 0 }
210 };
211
212 /*
213 * OS <-> Driver interface structures
214 */
215 MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations");
216
217 static device_method_t ntb_pci_methods[] = {
218 /* Device interface */
219 DEVMETHOD(device_probe, ntb_probe),
220 DEVMETHOD(device_attach, ntb_attach),
221 DEVMETHOD(device_detach, ntb_detach),
222 DEVMETHOD_END
223 };
224
225 static driver_t ntb_pci_driver = {
226 "ntb_hw",
227 ntb_pci_methods,
228 sizeof(struct ntb_softc),
229 };
230
231 static devclass_t ntb_devclass;
232 DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL);
233 MODULE_VERSION(ntb_hw, 1);
234
235 /*
236 * OS <-> Driver linkage functions
237 */
238 static int
ntb_probe(device_t device)239 ntb_probe(device_t device)
240 {
241 struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device));
242
243 if (p != NULL) {
244 device_set_desc(device, p->desc);
245 return (0);
246 } else
247 return (ENXIO);
248 }
249
250 #define DETACH_ON_ERROR(func) \
251 error = func; \
252 if (error < 0) { \
253 ntb_detach(device); \
254 return (error); \
255 }
256
257 static int
ntb_attach(device_t device)258 ntb_attach(device_t device)
259 {
260 struct ntb_softc *ntb = DEVICE2SOFTC(device);
261 struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device));
262 int error;
263
264 ntb->device = device;
265 ntb->type = p->type;
266 ntb->features = p->features;
267
268 /* Heartbeat timer for NTB_SOC since there is no link interrupt */
269 callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE);
270 callout_init(&ntb->lr_timer, CALLOUT_MPSAFE);
271
272 DETACH_ON_ERROR(ntb_map_pci_bars(ntb));
273 DETACH_ON_ERROR(ntb_initialize_hw(ntb));
274 DETACH_ON_ERROR(ntb_setup_interrupts(ntb));
275
276 pci_enable_busmaster(ntb->device);
277
278 return (error);
279 }
280
281 static int
ntb_detach(device_t device)282 ntb_detach(device_t device)
283 {
284 struct ntb_softc *ntb = DEVICE2SOFTC(device);
285
286 callout_drain(&ntb->heartbeat_timer);
287 callout_drain(&ntb->lr_timer);
288 ntb_teardown_interrupts(ntb);
289 ntb_unmap_pci_bar(ntb);
290
291 return (0);
292 }
293
294 static int
ntb_map_pci_bars(struct ntb_softc * ntb)295 ntb_map_pci_bars(struct ntb_softc *ntb)
296 {
297 int rc;
298
299 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0);
300 rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]);
301 if (rc != 0)
302 return rc;
303
304 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2);
305 rc = map_pci_bar(ntb, map_memory_window_bar,
306 &ntb->bar_info[NTB_B2B_BAR_1]);
307 if (rc != 0)
308 return rc;
309
310 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
311 if (HAS_FEATURE(NTB_REGS_THRU_MW))
312 rc = map_pci_bar(ntb, map_mmr_bar,
313 &ntb->bar_info[NTB_B2B_BAR_2]);
314 else
315 rc = map_pci_bar(ntb, map_memory_window_bar,
316 &ntb->bar_info[NTB_B2B_BAR_2]);
317 if (rc != 0)
318 return rc;
319
320 return (0);
321 }
322
323 static int
map_pci_bar(struct ntb_softc * ntb,bar_map_strategy strategy,struct ntb_pci_bar_info * bar)324 map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
325 struct ntb_pci_bar_info *bar)
326 {
327 int rc;
328
329 rc = strategy(ntb, bar);
330 if (rc != 0) {
331 device_printf(ntb->device,
332 "unable to allocate pci resource\n");
333 } else {
334 device_printf(ntb->device,
335 "Bar size = %lx, v %p, p %p\n",
336 bar->size, bar->vbase,
337 (void *)(bar->pbase));
338 }
339 return (rc);
340 }
341
342 static int
map_mmr_bar(struct ntb_softc * ntb,struct ntb_pci_bar_info * bar)343 map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
344 {
345
346 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
347 &bar->pci_resource_id, RF_ACTIVE);
348
349 if (bar->pci_resource == NULL)
350 return (ENXIO);
351 else {
352 save_bar_parameters(bar);
353 return (0);
354 }
355 }
356
357 static int
map_memory_window_bar(struct ntb_softc * ntb,struct ntb_pci_bar_info * bar)358 map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
359 {
360 int rc;
361 uint8_t bar_size_bits = 0;
362
363 bar->pci_resource = bus_alloc_resource_any(ntb->device,
364 SYS_RES_MEMORY, &bar->pci_resource_id, RF_ACTIVE);
365
366 if (bar->pci_resource == NULL)
367 return (ENXIO);
368 else {
369 save_bar_parameters(bar);
370 /*
371 * Ivytown NTB BAR sizes are misreported by the hardware due to
372 * a hardware issue. To work around this, query the size it
373 * should be configured to by the device and modify the resource
374 * to correspond to this new size. The BIOS on systems with this
375 * problem is required to provide enough address space to allow
376 * the driver to make this change safely.
377 *
378 * Ideally I could have just specified the size when I allocated
379 * the resource like:
380 * bus_alloc_resource(ntb->device,
381 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul,
382 * 1ul << bar_size_bits, RF_ACTIVE);
383 * but the PCI driver does not honor the size in this call, so
384 * we have to modify it after the fact.
385 */
386 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) {
387 if (bar->pci_resource_id == PCIR_BAR(2))
388 bar_size_bits = pci_read_config(ntb->device,
389 XEON_PBAR23SZ_OFFSET, 1);
390 else
391 bar_size_bits = pci_read_config(ntb->device,
392 XEON_PBAR45SZ_OFFSET, 1);
393 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY,
394 bar->pci_resource, bar->pbase,
395 bar->pbase + (1ul << bar_size_bits) - 1);
396 if (rc != 0 ) {
397 device_printf(ntb->device,
398 "unable to resize bar\n");
399 return (rc);
400 } else
401 save_bar_parameters(bar);
402 }
403
404 /* Mark bar region as write combining to improve performance. */
405 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size,
406 VM_MEMATTR_WRITE_COMBINING);
407 if (rc != 0) {
408 device_printf(ntb->device, "unable to mark bar as"
409 " WRITE_COMBINING\n");
410 return (rc);
411 }
412 }
413 return (0);
414 }
415
416 static void
ntb_unmap_pci_bar(struct ntb_softc * ntb)417 ntb_unmap_pci_bar(struct ntb_softc *ntb)
418 {
419 struct ntb_pci_bar_info *current_bar;
420 int i;
421
422 for (i = 0; i< NTB_MAX_BARS; i++) {
423 current_bar = &ntb->bar_info[i];
424 if (current_bar->pci_resource != NULL)
425 bus_release_resource(ntb->device, SYS_RES_MEMORY,
426 current_bar->pci_resource_id,
427 current_bar->pci_resource);
428 }
429 }
430
431 static int
ntb_setup_interrupts(struct ntb_softc * ntb)432 ntb_setup_interrupts(struct ntb_softc *ntb)
433 {
434 void (*interrupt_handler)(void *);
435 void *int_arg;
436 bool use_msix = 0;
437 uint32_t num_vectors;
438 int i;
439
440 ntb->allocated_interrupts = 0;
441 /*
442 * On SOC, disable all interrupts. On XEON, disable all but Link
443 * Interrupt. The rest will be unmasked as callbacks are registered.
444 */
445 if (ntb->type == NTB_SOC)
446 ntb_reg_write(8, ntb->reg_ofs.pdb_mask, ~0);
447 else
448 ntb_reg_write(2, ntb->reg_ofs.pdb_mask,
449 ~(1 << ntb->limits.max_db_bits));
450
451 num_vectors = MIN(pci_msix_count(ntb->device),
452 ntb->limits.max_db_bits);
453 if (num_vectors >= 1) {
454 pci_alloc_msix(ntb->device, &num_vectors);
455 if (num_vectors >= 4)
456 use_msix = TRUE;
457 }
458
459 ntb_create_callbacks(ntb, num_vectors);
460 if (use_msix == TRUE) {
461 for (i = 0; i < num_vectors; i++) {
462 ntb->int_info[i].rid = i + 1;
463 ntb->int_info[i].res = bus_alloc_resource_any(
464 ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid,
465 RF_ACTIVE);
466 if (ntb->int_info[i].res == NULL) {
467 device_printf(ntb->device,
468 "bus_alloc_resource failed\n");
469 return (-1);
470 }
471 ntb->int_info[i].tag = NULL;
472 ntb->allocated_interrupts++;
473 if (ntb->type == NTB_SOC) {
474 interrupt_handler = handle_soc_irq;
475 int_arg = &ntb->db_cb[i];
476 } else {
477 if (i == num_vectors - 1) {
478 interrupt_handler =
479 handle_xeon_event_irq;
480 int_arg = ntb;
481 } else {
482 interrupt_handler =
483 handle_xeon_irq;
484 int_arg = &ntb->db_cb[i];
485 }
486 }
487 if (bus_setup_intr(ntb->device, ntb->int_info[i].res,
488 INTR_MPSAFE | INTR_TYPE_MISC, NULL,
489 interrupt_handler, int_arg,
490 &ntb->int_info[i].tag) != 0) {
491 device_printf(ntb->device,
492 "bus_setup_intr failed\n");
493 return (ENXIO);
494 }
495 }
496 }
497 else {
498 ntb->int_info[0].rid = 0;
499 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device,
500 SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
501 interrupt_handler = ntb_handle_legacy_interrupt;
502 if (ntb->int_info[0].res == NULL) {
503 device_printf(ntb->device,
504 "bus_alloc_resource failed\n");
505 return (-1);
506 }
507 ntb->int_info[0].tag = NULL;
508 ntb->allocated_interrupts = 1;
509
510 if (bus_setup_intr(ntb->device, ntb->int_info[0].res,
511 INTR_MPSAFE | INTR_TYPE_MISC, NULL,
512 interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) {
513
514 device_printf(ntb->device, "bus_setup_intr failed\n");
515 return (ENXIO);
516 }
517 }
518
519 return (0);
520 }
521
522 static void
ntb_teardown_interrupts(struct ntb_softc * ntb)523 ntb_teardown_interrupts(struct ntb_softc *ntb)
524 {
525 struct ntb_int_info *current_int;
526 int i;
527
528 for (i=0; i<ntb->allocated_interrupts; i++) {
529 current_int = &ntb->int_info[i];
530 if (current_int->tag != NULL)
531 bus_teardown_intr(ntb->device, current_int->res,
532 current_int->tag);
533
534 if (current_int->res != NULL)
535 bus_release_resource(ntb->device, SYS_RES_IRQ,
536 rman_get_rid(current_int->res), current_int->res);
537 }
538
539 ntb_free_callbacks(ntb);
540 pci_release_msi(ntb->device);
541 }
542
543 static void
handle_soc_irq(void * arg)544 handle_soc_irq(void *arg)
545 {
546 struct ntb_db_cb *db_cb = arg;
547 struct ntb_softc *ntb = db_cb->ntb;
548
549 ntb_reg_write(8, ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num);
550
551 if (db_cb->callback != NULL)
552 db_cb->callback(db_cb->data, db_cb->db_num);
553 }
554
555 static void
handle_xeon_irq(void * arg)556 handle_xeon_irq(void *arg)
557 {
558 struct ntb_db_cb *db_cb = arg;
559 struct ntb_softc *ntb = db_cb->ntb;
560
561 /*
562 * On Xeon, there are 16 bits in the interrupt register
563 * but only 4 vectors. So, 5 bits are assigned to the first 3
564 * vectors, with the 4th having a single bit for link
565 * interrupts.
566 */
567 ntb_reg_write(2, ntb->reg_ofs.pdb,
568 ((1 << ntb->bits_per_vector) - 1) <<
569 (db_cb->db_num * ntb->bits_per_vector));
570
571 if (db_cb->callback != NULL)
572 db_cb->callback(db_cb->data, db_cb->db_num);
573 }
574
575 /* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */
576 static void
handle_xeon_event_irq(void * arg)577 handle_xeon_event_irq(void *arg)
578 {
579 struct ntb_softc *ntb = arg;
580 int rc;
581
582 rc = ntb_check_link_status(ntb);
583 if (rc != 0)
584 device_printf(ntb->device, "Error determining link status\n");
585
586 /* bit 15 is always the link bit */
587 ntb_reg_write(2, ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits);
588 }
589
590 static void
ntb_handle_legacy_interrupt(void * arg)591 ntb_handle_legacy_interrupt(void *arg)
592 {
593 struct ntb_softc *ntb = arg;
594 unsigned int i = 0;
595 uint64_t pdb64;
596 uint16_t pdb16;
597
598 if (ntb->type == NTB_SOC) {
599 pdb64 = ntb_reg_read(8, ntb->reg_ofs.pdb);
600
601 while (pdb64) {
602 i = ffs(pdb64);
603 pdb64 &= pdb64 - 1;
604 handle_soc_irq(&ntb->db_cb[i]);
605 }
606 } else {
607 pdb16 = ntb_reg_read(2, ntb->reg_ofs.pdb);
608
609 if ((pdb16 & XEON_DB_HW_LINK) != 0) {
610 handle_xeon_event_irq(ntb);
611 pdb16 &= ~XEON_DB_HW_LINK;
612 }
613
614 while (pdb16 != 0) {
615 i = ffs(pdb16);
616 pdb16 &= pdb16 - 1;
617 handle_xeon_irq(&ntb->db_cb[i]);
618 }
619 }
620
621 }
622
623 static int
ntb_create_callbacks(struct ntb_softc * ntb,int num_vectors)624 ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors)
625 {
626 int i;
627
628 ntb->db_cb = malloc(num_vectors * sizeof(struct ntb_db_cb), M_NTB,
629 M_ZERO | M_WAITOK);
630 for (i = 0; i < num_vectors; i++) {
631 ntb->db_cb[i].db_num = i;
632 ntb->db_cb[i].ntb = ntb;
633 }
634
635 return (0);
636 }
637
638 static void
ntb_free_callbacks(struct ntb_softc * ntb)639 ntb_free_callbacks(struct ntb_softc *ntb)
640 {
641 int i;
642
643 for (i = 0; i < ntb->limits.max_db_bits; i++)
644 ntb_unregister_db_callback(ntb, i);
645
646 free(ntb->db_cb, M_NTB);
647 }
648
649 static struct ntb_hw_info *
ntb_get_device_info(uint32_t device_id)650 ntb_get_device_info(uint32_t device_id)
651 {
652 struct ntb_hw_info *ep = pci_ids;
653
654 while (ep->device_id) {
655 if (ep->device_id == device_id)
656 return (ep);
657 ++ep;
658 }
659 return (NULL);
660 }
661
662 static int
ntb_initialize_hw(struct ntb_softc * ntb)663 ntb_initialize_hw(struct ntb_softc *ntb)
664 {
665
666 if (ntb->type == NTB_SOC)
667 return (ntb_setup_soc(ntb));
668 else
669 return (ntb_setup_xeon(ntb));
670 }
671
672 static int
ntb_setup_xeon(struct ntb_softc * ntb)673 ntb_setup_xeon(struct ntb_softc *ntb)
674 {
675 uint8_t val, connection_type;
676
677 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
678
679 connection_type = val & XEON_PPD_CONN_TYPE;
680 switch (connection_type) {
681 case NTB_CONN_B2B:
682 ntb->conn_type = NTB_CONN_B2B;
683 break;
684 case NTB_CONN_CLASSIC:
685 case NTB_CONN_RP:
686 default:
687 device_printf(ntb->device, "Connection type %d not supported\n",
688 connection_type);
689 return (ENXIO);
690 }
691
692 if ((val & XEON_PPD_DEV_TYPE) != 0)
693 ntb->dev_type = NTB_DEV_DSD;
694 else
695 ntb->dev_type = NTB_DEV_USD;
696
697 ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET;
698 ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET;
699 ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET;
700 ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET;
701 ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET;
702 ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET;
703 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET;
704 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET;
705
706 if (ntb->conn_type == NTB_CONN_B2B) {
707 ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET;
708 ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
709 ntb->limits.max_spads = XEON_MAX_SPADS;
710 } else {
711 ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET;
712 ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET;
713 ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS;
714 }
715
716 ntb->limits.max_db_bits = XEON_MAX_DB_BITS;
717 ntb->limits.msix_cnt = XEON_MSIX_CNT;
718 ntb->bits_per_vector = XEON_DB_BITS_PER_VEC;
719
720 configure_xeon_secondary_side_bars(ntb);
721 /* Enable Bus Master and Memory Space on the secondary side */
722 ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
723 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
724
725 /* Enable link training */
726 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl,
727 NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);
728
729 return (0);
730 }
731
732 static int
ntb_setup_soc(struct ntb_softc * ntb)733 ntb_setup_soc(struct ntb_softc *ntb)
734 {
735 uint32_t val, connection_type;
736
737 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
738
739 connection_type = (val & SOC_PPD_CONN_TYPE) >> 8;
740 switch (connection_type) {
741 case NTB_CONN_B2B:
742 ntb->conn_type = NTB_CONN_B2B;
743 break;
744 case NTB_CONN_RP:
745 default:
746 device_printf(ntb->device, "Connection type %d not supported\n",
747 connection_type);
748 return (ENXIO);
749 }
750
751 if ((val & SOC_PPD_DEV_TYPE) != 0)
752 ntb->dev_type = NTB_DEV_DSD;
753 else
754 ntb->dev_type = NTB_DEV_USD;
755
756 /* Initiate PCI-E link training */
757 pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK,
758 4);
759
760 ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET;
761 ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET;
762 ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET;
763 ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET;
764 ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET;
765 ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET;
766 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET;
767 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET;
768
769 if (ntb->conn_type == NTB_CONN_B2B) {
770 ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET;
771 ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET;
772 ntb->limits.max_spads = SOC_MAX_SPADS;
773 } else {
774 ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET;
775 ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET;
776 ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS;
777 }
778
779 ntb->limits.max_db_bits = SOC_MAX_DB_BITS;
780 ntb->limits.msix_cnt = SOC_MSIX_CNT;
781 ntb->bits_per_vector = SOC_DB_BITS_PER_VEC;
782
783 /*
784 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is
785 * resolved. Mask transaction layer internal parity errors.
786 */
787 pci_write_config(ntb->device, 0xFC, 0x4, 4);
788
789 configure_soc_secondary_side_bars(ntb);
790
791 /* Enable Bus Master and Memory Space on the secondary side */
792 ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
793 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
794 callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb);
795
796 return (0);
797 }
798
799 static void
configure_soc_secondary_side_bars(struct ntb_softc * ntb)800 configure_soc_secondary_side_bars(struct ntb_softc *ntb)
801 {
802
803 if (ntb->dev_type == NTB_DEV_USD) {
804 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
805 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR);
806 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR);
807 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR);
808 } else {
809 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR);
810 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR);
811 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR);
812 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR);
813 }
814 }
815
816 static void
configure_xeon_secondary_side_bars(struct ntb_softc * ntb)817 configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
818 {
819
820 if (ntb->dev_type == NTB_DEV_USD) {
821 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
822 if (HAS_FEATURE(NTB_REGS_THRU_MW))
823 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
824 MBAR01_DSD_ADDR);
825 else
826 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
827 PBAR4XLAT_USD_ADDR);
828 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR);
829 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR);
830 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR);
831 } else {
832 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR);
833 if (HAS_FEATURE(NTB_REGS_THRU_MW))
834 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
835 MBAR01_USD_ADDR);
836 else
837 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
838 PBAR4XLAT_DSD_ADDR);
839 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR);
840 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR);
841 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR);
842 }
843 }
844
845 /* SOC does not have link status interrupt, poll on that platform */
846 static void
ntb_handle_heartbeat(void * arg)847 ntb_handle_heartbeat(void *arg)
848 {
849 struct ntb_softc *ntb = arg;
850 uint32_t status32;
851 int rc = ntb_check_link_status(ntb);
852
853 if (rc != 0)
854 device_printf(ntb->device,
855 "Error determining link status\n");
856 /* Check to see if a link error is the cause of the link down */
857 if (ntb->link_status == NTB_LINK_DOWN) {
858 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
859 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) {
860 callout_reset(&ntb->lr_timer, 0, recover_soc_link,
861 ntb);
862 return;
863 }
864 }
865
866 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
867 ntb_handle_heartbeat, ntb);
868 }
869
870 static void
soc_perform_link_restart(struct ntb_softc * ntb)871 soc_perform_link_restart(struct ntb_softc *ntb)
872 {
873 uint32_t status;
874
875 /* Driver resets the NTB ModPhy lanes - magic! */
876 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0);
877 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40);
878 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60);
879 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60);
880
881 /* Driver waits 100ms to allow the NTB ModPhy to settle */
882 pause("ModPhy", hz / 10);
883
884 /* Clear AER Errors, write to clear */
885 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET);
886 status &= PCIM_AER_COR_REPLAY_ROLLOVER;
887 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status);
888
889 /* Clear unexpected electrical idle event in LTSSM, write to clear */
890 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET);
891 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI;
892 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status);
893
894 /* Clear DeSkew Buffer error, write to clear */
895 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET);
896 status |= SOC_DESKEWSTS_DBERR;
897 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status);
898
899 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
900 status &= SOC_IBIST_ERR_OFLOW;
901 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status);
902
903 /* Releases the NTB state machine to allow the link to retrain */
904 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
905 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT;
906 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status);
907 }
908
909 static void
ntb_handle_link_event(struct ntb_softc * ntb,int link_state)910 ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
911 {
912 enum ntb_hw_event event;
913 uint16_t status;
914
915 if (ntb->link_status == link_state)
916 return;
917
918 if (link_state == NTB_LINK_UP) {
919 device_printf(ntb->device, "Link Up\n");
920 ntb->link_status = NTB_LINK_UP;
921 event = NTB_EVENT_HW_LINK_UP;
922
923 if (ntb->type == NTB_SOC)
924 status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat);
925 else
926 status = pci_read_config(ntb->device,
927 XEON_LINK_STATUS_OFFSET, 2);
928 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
929 ntb->link_speed = (status & NTB_LINK_SPEED_MASK);
930 device_printf(ntb->device, "Link Width %d, Link Speed %d\n",
931 ntb->link_width, ntb->link_speed);
932 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
933 ntb_handle_heartbeat, ntb);
934 } else {
935 device_printf(ntb->device, "Link Down\n");
936 ntb->link_status = NTB_LINK_DOWN;
937 event = NTB_EVENT_HW_LINK_DOWN;
938 /* Do not modify link width/speed, we need it in link recovery */
939 }
940
941 /* notify the upper layer if we have an event change */
942 if (ntb->event_cb != NULL)
943 ntb->event_cb(ntb->ntb_transport, event);
944 }
945
946 static void
recover_soc_link(void * arg)947 recover_soc_link(void *arg)
948 {
949 struct ntb_softc *ntb = arg;
950 uint8_t speed, width;
951 uint32_t status32;
952 uint16_t status16;
953
954 soc_perform_link_restart(ntb);
955 pause("Link", SOC_LINK_RECOVERY_TIME * hz / 1000);
956
957 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
958 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0)
959 goto retry;
960
961 status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
962 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0)
963 goto retry;
964
965 status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat);
966 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
967 speed = (status16 & NTB_LINK_SPEED_MASK);
968 if (ntb->link_width != width || ntb->link_speed != speed)
969 goto retry;
970
971 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
972 ntb_handle_heartbeat, ntb);
973 return;
974
975 retry:
976 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link,
977 ntb);
978 }
979
980 static int
ntb_check_link_status(struct ntb_softc * ntb)981 ntb_check_link_status(struct ntb_softc *ntb)
982 {
983 int link_state;
984 uint32_t ntb_cntl;
985 uint16_t status;
986
987 if (ntb->type == NTB_SOC) {
988 ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
989 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0)
990 link_state = NTB_LINK_DOWN;
991 else
992 link_state = NTB_LINK_UP;
993 } else {
994 status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET,
995 2);
996
997 if ((status & NTB_LINK_STATUS_ACTIVE) != 0)
998 link_state = NTB_LINK_UP;
999 else
1000 link_state = NTB_LINK_DOWN;
1001 }
1002
1003 ntb_handle_link_event(ntb, link_state);
1004
1005 return (0);
1006 }
1007
1008 /**
1009 * ntb_register_event_callback() - register event callback
1010 * @ntb: pointer to ntb_softc instance
1011 * @func: callback function to register
1012 *
1013 * This function registers a callback for any HW driver events such as link
1014 * up/down, power management notices and etc.
1015 *
1016 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1017 */
1018 int
ntb_register_event_callback(struct ntb_softc * ntb,ntb_event_callback func)1019 ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func)
1020 {
1021
1022 if (ntb->event_cb != NULL)
1023 return (EINVAL);
1024
1025 ntb->event_cb = func;
1026
1027 return (0);
1028 }
1029
1030 /**
1031 * ntb_unregister_event_callback() - unregisters the event callback
1032 * @ntb: pointer to ntb_softc instance
1033 *
1034 * This function unregisters the existing callback from transport
1035 */
1036 void
ntb_unregister_event_callback(struct ntb_softc * ntb)1037 ntb_unregister_event_callback(struct ntb_softc *ntb)
1038 {
1039
1040 ntb->event_cb = NULL;
1041 }
1042
1043 /**
1044 * ntb_register_db_callback() - register a callback for doorbell interrupt
1045 * @ntb: pointer to ntb_softc instance
1046 * @idx: doorbell index to register callback, zero based
1047 * @func: callback function to register
1048 *
1049 * This function registers a callback function for the doorbell interrupt
1050 * on the primary side. The function will unmask the doorbell as well to
1051 * allow interrupt.
1052 *
1053 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1054 */
1055 int
ntb_register_db_callback(struct ntb_softc * ntb,unsigned int idx,void * data,ntb_db_callback func)1056 ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
1057 ntb_db_callback func)
1058 {
1059 uint16_t mask;
1060
1061 if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) {
1062 device_printf(ntb->device, "Invalid Index.\n");
1063 return (EINVAL);
1064 }
1065
1066 ntb->db_cb[idx].callback = func;
1067 ntb->db_cb[idx].data = data;
1068
1069 /* unmask interrupt */
1070 mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask);
1071 mask &= ~(1 << (idx * ntb->bits_per_vector));
1072 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask);
1073
1074 return (0);
1075 }
1076
1077 /**
1078 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
1079 * @ntb: pointer to ntb_softc instance
1080 * @idx: doorbell index to register callback, zero based
1081 *
1082 * This function unregisters a callback function for the doorbell interrupt
1083 * on the primary side. The function will also mask the said doorbell.
1084 */
1085 void
ntb_unregister_db_callback(struct ntb_softc * ntb,unsigned int idx)1086 ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx)
1087 {
1088 unsigned long mask;
1089
1090 if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback)
1091 return;
1092
1093 mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask);
1094 mask |= 1 << (idx * ntb->bits_per_vector);
1095 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask);
1096
1097 ntb->db_cb[idx].callback = NULL;
1098 }
1099
1100 /**
1101 * ntb_find_transport() - find the transport pointer
1102 * @transport: pointer to pci device
1103 *
1104 * Given the pci device pointer, return the transport pointer passed in when
1105 * the transport attached when it was inited.
1106 *
1107 * RETURNS: pointer to transport.
1108 */
1109 void *
ntb_find_transport(struct ntb_softc * ntb)1110 ntb_find_transport(struct ntb_softc *ntb)
1111 {
1112
1113 return (ntb->ntb_transport);
1114 }
1115
1116 /**
1117 * ntb_register_transport() - Register NTB transport with NTB HW driver
1118 * @transport: transport identifier
1119 *
1120 * This function allows a transport to reserve the hardware driver for
1121 * NTB usage.
1122 *
1123 * RETURNS: pointer to ntb_softc, NULL on error.
1124 */
1125 struct ntb_softc *
ntb_register_transport(struct ntb_softc * ntb,void * transport)1126 ntb_register_transport(struct ntb_softc *ntb, void *transport)
1127 {
1128
1129 /*
1130 * TODO: when we have more than one transport, we will need to rewrite
1131 * this to prevent race conditions
1132 */
1133 if (ntb->ntb_transport != NULL)
1134 return (NULL);
1135
1136 ntb->ntb_transport = transport;
1137 return (ntb);
1138 }
1139
1140 /**
1141 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
1142 * @ntb - ntb_softc of the transport to be freed
1143 *
1144 * This function unregisters the transport from the HW driver and performs any
1145 * necessary cleanups.
1146 */
1147 void
ntb_unregister_transport(struct ntb_softc * ntb)1148 ntb_unregister_transport(struct ntb_softc *ntb)
1149 {
1150 int i;
1151
1152 if (ntb->ntb_transport == NULL)
1153 return;
1154
1155 for (i = 0; i < ntb->allocated_interrupts; i++)
1156 ntb_unregister_db_callback(ntb, i);
1157
1158 ntb_unregister_event_callback(ntb);
1159 ntb->ntb_transport = NULL;
1160 }
1161
1162 /**
1163 * ntb_get_max_spads() - get the total scratch regs usable
1164 * @ntb: pointer to ntb_softc instance
1165 *
1166 * This function returns the max 32bit scratchpad registers usable by the
1167 * upper layer.
1168 *
1169 * RETURNS: total number of scratch pad registers available
1170 */
1171 int
ntb_get_max_spads(struct ntb_softc * ntb)1172 ntb_get_max_spads(struct ntb_softc *ntb)
1173 {
1174
1175 return (ntb->limits.max_spads);
1176 }
1177
1178 /**
1179 * ntb_write_local_spad() - write to the secondary scratchpad register
1180 * @ntb: pointer to ntb_softc instance
1181 * @idx: index to the scratchpad register, 0 based
1182 * @val: the data value to put into the register
1183 *
1184 * This function allows writing of a 32bit value to the indexed scratchpad
1185 * register. The register resides on the secondary (external) side.
1186 *
1187 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1188 */
1189 int
ntb_write_local_spad(struct ntb_softc * ntb,unsigned int idx,uint32_t val)1190 ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
1191 {
1192
1193 if (idx >= ntb->limits.max_spads)
1194 return (EINVAL);
1195
1196 ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val);
1197
1198 return (0);
1199 }
1200
1201 /**
1202 * ntb_read_local_spad() - read from the primary scratchpad register
1203 * @ntb: pointer to ntb_softc instance
1204 * @idx: index to scratchpad register, 0 based
1205 * @val: pointer to 32bit integer for storing the register value
1206 *
1207 * This function allows reading of the 32bit scratchpad register on
1208 * the primary (internal) side.
1209 *
1210 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1211 */
1212 int
ntb_read_local_spad(struct ntb_softc * ntb,unsigned int idx,uint32_t * val)1213 ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
1214 {
1215
1216 if (idx >= ntb->limits.max_spads)
1217 return (EINVAL);
1218
1219 *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4);
1220
1221 return (0);
1222 }
1223
1224 /**
1225 * ntb_write_remote_spad() - write to the secondary scratchpad register
1226 * @ntb: pointer to ntb_softc instance
1227 * @idx: index to the scratchpad register, 0 based
1228 * @val: the data value to put into the register
1229 *
1230 * This function allows writing of a 32bit value to the indexed scratchpad
1231 * register. The register resides on the secondary (external) side.
1232 *
1233 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1234 */
1235 int
ntb_write_remote_spad(struct ntb_softc * ntb,unsigned int idx,uint32_t val)1236 ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
1237 {
1238
1239 if (idx >= ntb->limits.max_spads)
1240 return (EINVAL);
1241
1242 if (HAS_FEATURE(NTB_REGS_THRU_MW))
1243 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val);
1244 else
1245 ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val);
1246
1247 return (0);
1248 }
1249
1250 /**
1251 * ntb_read_remote_spad() - read from the primary scratchpad register
1252 * @ntb: pointer to ntb_softc instance
1253 * @idx: index to scratchpad register, 0 based
1254 * @val: pointer to 32bit integer for storing the register value
1255 *
1256 * This function allows reading of the 32bit scratchpad register on
1257 * the primary (internal) side.
1258 *
1259 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1260 */
1261 int
ntb_read_remote_spad(struct ntb_softc * ntb,unsigned int idx,uint32_t * val)1262 ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
1263 {
1264
1265 if (idx >= ntb->limits.max_spads)
1266 return (EINVAL);
1267
1268 if (HAS_FEATURE(NTB_REGS_THRU_MW))
1269 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4);
1270 else
1271 *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4);
1272
1273 return (0);
1274 }
1275
1276 /**
1277 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
1278 * @ntb: pointer to ntb_softc instance
1279 * @mw: memory window number
1280 *
1281 * This function provides the base virtual address of the memory window
1282 * specified.
1283 *
1284 * RETURNS: pointer to virtual address, or NULL on error.
1285 */
1286 void *
ntb_get_mw_vbase(struct ntb_softc * ntb,unsigned int mw)1287 ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw)
1288 {
1289
1290 if (mw >= NTB_NUM_MW)
1291 return (NULL);
1292
1293 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase);
1294 }
1295
1296 vm_paddr_t
ntb_get_mw_pbase(struct ntb_softc * ntb,unsigned int mw)1297 ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw)
1298 {
1299
1300 if (mw >= NTB_NUM_MW)
1301 return (0);
1302
1303 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase);
1304 }
1305
1306 /**
1307 * ntb_get_mw_size() - return size of NTB memory window
1308 * @ntb: pointer to ntb_softc instance
1309 * @mw: memory window number
1310 *
1311 * This function provides the physical size of the memory window specified
1312 *
1313 * RETURNS: the size of the memory window or zero on error
1314 */
1315 u_long
ntb_get_mw_size(struct ntb_softc * ntb,unsigned int mw)1316 ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw)
1317 {
1318
1319 if (mw >= NTB_NUM_MW)
1320 return (0);
1321
1322 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size);
1323 }
1324
1325 /**
1326 * ntb_set_mw_addr - set the memory window address
1327 * @ntb: pointer to ntb_softc instance
1328 * @mw: memory window number
1329 * @addr: base address for data
1330 *
1331 * This function sets the base physical address of the memory window. This
1332 * memory address is where data from the remote system will be transfered into
1333 * or out of depending on how the transport is configured.
1334 */
1335 void
ntb_set_mw_addr(struct ntb_softc * ntb,unsigned int mw,uint64_t addr)1336 ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
1337 {
1338
1339 if (mw >= NTB_NUM_MW)
1340 return;
1341
1342 switch (NTB_MW_TO_BAR(mw)) {
1343 case NTB_B2B_BAR_1:
1344 ntb_reg_write(8, ntb->reg_ofs.sbar2_xlat, addr);
1345 break;
1346 case NTB_B2B_BAR_2:
1347 ntb_reg_write(8, ntb->reg_ofs.sbar4_xlat, addr);
1348 break;
1349 }
1350 }
1351
1352 /**
1353 * ntb_ring_sdb() - Set the doorbell on the secondary/external side
1354 * @ntb: pointer to ntb_softc instance
1355 * @db: doorbell to ring
1356 *
1357 * This function allows triggering of a doorbell on the secondary/external
1358 * side that will initiate an interrupt on the remote host
1359 *
1360 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
1361 */
1362 void
ntb_ring_sdb(struct ntb_softc * ntb,unsigned int db)1363 ntb_ring_sdb(struct ntb_softc *ntb, unsigned int db)
1364 {
1365
1366 if (ntb->type == NTB_SOC)
1367 ntb_reg_write(8, ntb->reg_ofs.sdb, (uint64_t) 1 << db);
1368 else
1369 if (HAS_FEATURE(NTB_REGS_THRU_MW))
1370 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET,
1371 ((1 << ntb->bits_per_vector) - 1) <<
1372 (db * ntb->bits_per_vector));
1373 else
1374 ntb_reg_write(2, ntb->reg_ofs.sdb,
1375 ((1 << ntb->bits_per_vector) - 1) <<
1376 (db * ntb->bits_per_vector));
1377 }
1378
1379 /**
1380 * ntb_query_link_status() - return the hardware link status
1381 * @ndev: pointer to ntb_device instance
1382 *
1383 * Returns true if the hardware is connected to the remote system
1384 *
1385 * RETURNS: true or false based on the hardware link state
1386 */
1387 bool
ntb_query_link_status(struct ntb_softc * ntb)1388 ntb_query_link_status(struct ntb_softc *ntb)
1389 {
1390
1391 return (ntb->link_status == NTB_LINK_UP);
1392 }
1393
1394 static void
save_bar_parameters(struct ntb_pci_bar_info * bar)1395 save_bar_parameters(struct ntb_pci_bar_info *bar)
1396 {
1397 bar->pci_bus_tag =
1398 rman_get_bustag(bar->pci_resource);
1399 bar->pci_bus_handle =
1400 rman_get_bushandle(bar->pci_resource);
1401 bar->pbase =
1402 rman_get_start(bar->pci_resource);
1403 bar->size =
1404 rman_get_size(bar->pci_resource);
1405 bar->vbase =
1406 rman_get_virtual(bar->pci_resource);
1407
1408 }
1409
ntb_get_device(struct ntb_softc * ntb)1410 device_t ntb_get_device(struct ntb_softc *ntb)
1411 {
1412
1413 return (ntb->device);
1414 }
1415