1 /*
2 * BSD LICENSE
3 *
4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Cavium, Inc. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include "lio_bsd.h"
35 #include "lio_common.h"
36 #include "lio_droq.h"
37 #include "lio_iq.h"
38 #include "lio_response_manager.h"
39 #include "lio_device.h"
40 #include "lio_main.h"
41 #include "lio_network.h"
42 #include "cn23xx_pf_device.h"
43 #include "lio_rxtx.h"
44
45 struct lio_iq_post_status {
46 int status;
47 int index;
48 };
49
50 static void lio_check_db_timeout(void *arg, int pending);
51 static void __lio_check_db_timeout(struct octeon_device *oct,
52 uint64_t iq_no);
53
54 /* Return 0 on success, 1 on failure */
55 int
lio_init_instr_queue(struct octeon_device * oct,union octeon_txpciq txpciq,uint32_t num_descs)56 lio_init_instr_queue(struct octeon_device *oct, union octeon_txpciq txpciq,
57 uint32_t num_descs)
58 {
59 struct lio_instr_queue *iq;
60 struct lio_iq_config *conf = NULL;
61 struct lio_tq *db_tq;
62 struct lio_request_list *request_buf;
63 bus_size_t max_size;
64 uint32_t iq_no = (uint32_t)txpciq.s.q_no;
65 uint32_t q_size;
66 int error, i;
67
68 if (LIO_CN23XX_PF(oct))
69 conf = &(LIO_GET_IQ_CFG(LIO_CHIP_CONF(oct, cn23xx_pf)));
70 if (conf == NULL) {
71 lio_dev_err(oct, "Unsupported Chip %x\n", oct->chip_id);
72 return (1);
73 }
74
75 q_size = (uint32_t)conf->instr_type * num_descs;
76 iq = oct->instr_queue[iq_no];
77 iq->oct_dev = oct;
78
79 max_size = LIO_CN23XX_PKI_MAX_FRAME_SIZE * num_descs;
80
81 error = bus_dma_tag_create(bus_get_dma_tag(oct->device), /* parent */
82 1, 0, /* alignment, bounds */
83 BUS_SPACE_MAXADDR, /* lowaddr */
84 BUS_SPACE_MAXADDR, /* highaddr */
85 NULL, NULL, /* filter, filterarg */
86 max_size, /* maxsize */
87 LIO_MAX_SG, /* nsegments */
88 PAGE_SIZE, /* maxsegsize */
89 0, /* flags */
90 NULL, /* lockfunc */
91 NULL, /* lockfuncarg */
92 &iq->txtag);
93 if (error) {
94 lio_dev_err(oct, "Cannot allocate memory for instr queue %d\n",
95 iq_no);
96 return (1);
97 }
98
99 iq->base_addr = lio_dma_alloc(q_size, (vm_paddr_t *)&iq->base_addr_dma);
100 if (!iq->base_addr) {
101 lio_dev_err(oct, "Cannot allocate memory for instr queue %d\n",
102 iq_no);
103 return (1);
104 }
105
106 iq->max_count = num_descs;
107
108 /*
109 * Initialize a list to holds requests that have been posted to
110 * Octeon but has yet to be fetched by octeon
111 */
112 iq->request_list = malloc(sizeof(*iq->request_list) * num_descs,
113 M_DEVBUF, M_NOWAIT | M_ZERO);
114 if (iq->request_list == NULL) {
115 lio_dev_err(oct, "Alloc failed for IQ[%d] nr free list\n",
116 iq_no);
117 return (1);
118 }
119
120 lio_dev_dbg(oct, "IQ[%d]: base: %p basedma: %llx count: %d\n",
121 iq_no, iq->base_addr, LIO_CAST64(iq->base_addr_dma),
122 iq->max_count);
123
124 /* Create the descriptor buffer dma maps */
125 request_buf = iq->request_list;
126 for (i = 0; i < num_descs; i++, request_buf++) {
127 error = bus_dmamap_create(iq->txtag, 0, &request_buf->map);
128 if (error) {
129 lio_dev_err(oct, "Unable to create TX DMA map\n");
130 return (1);
131 }
132 }
133
134 iq->txpciq.txpciq64 = txpciq.txpciq64;
135 iq->fill_cnt = 0;
136 iq->host_write_index = 0;
137 iq->octeon_read_index = 0;
138 iq->flush_index = 0;
139 iq->last_db_time = 0;
140 iq->db_timeout = (uint32_t)conf->db_timeout;
141 atomic_store_rel_int(&iq->instr_pending, 0);
142
143 /* Initialize the lock for this instruction queue */
144 mtx_init(&iq->lock, "Tx_lock", NULL, MTX_DEF);
145 mtx_init(&iq->post_lock, "iq_post_lock", NULL, MTX_DEF);
146 mtx_init(&iq->enq_lock, "enq_lock", NULL, MTX_DEF);
147
148 mtx_init(&iq->iq_flush_running_lock, "iq_flush_running_lock", NULL,
149 MTX_DEF);
150
151 oct->io_qmask.iq |= BIT_ULL(iq_no);
152
153 /* Set the 32B/64B mode for each input queue */
154 oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no);
155 iq->iqcmd_64B = (conf->instr_type == 64);
156
157 oct->fn_list.setup_iq_regs(oct, iq_no);
158
159 db_tq = &oct->check_db_tq[iq_no];
160 db_tq->tq = taskqueue_create("lio_check_db_timeout", M_WAITOK,
161 taskqueue_thread_enqueue, &db_tq->tq);
162
163 TIMEOUT_TASK_INIT(db_tq->tq, &db_tq->work, 0, lio_check_db_timeout,
164 (void *)db_tq);
165 db_tq->ctxul = iq_no;
166 db_tq->ctxptr = oct;
167
168 taskqueue_start_threads(&db_tq->tq, 1, PI_NET,
169 "lio%d_check_db_timeout:%d",
170 oct->octeon_id, iq_no);
171 taskqueue_enqueue_timeout(db_tq->tq, &db_tq->work, 1);
172
173 /* Allocate a buf ring */
174 oct->instr_queue[iq_no]->br =
175 buf_ring_alloc(LIO_BR_SIZE, M_DEVBUF, M_WAITOK,
176 &oct->instr_queue[iq_no]->enq_lock);
177
178 return (0);
179 }
180
181 int
lio_delete_instr_queue(struct octeon_device * oct,uint32_t iq_no)182 lio_delete_instr_queue(struct octeon_device *oct, uint32_t iq_no)
183 {
184 struct lio_instr_queue *iq = oct->instr_queue[iq_no];
185 struct lio_request_list *request_buf;
186 struct lio_mbuf_free_info *finfo;
187 uint64_t desc_size = 0, q_size;
188 int i;
189
190 lio_dev_dbg(oct, "%s[%d]\n", __func__, iq_no);
191
192 if (oct->check_db_tq[iq_no].tq != NULL) {
193 while (taskqueue_cancel_timeout(oct->check_db_tq[iq_no].tq,
194 &oct->check_db_tq[iq_no].work,
195 NULL))
196 taskqueue_drain_timeout(oct->check_db_tq[iq_no].tq,
197 &oct->check_db_tq[iq_no].work);
198 taskqueue_free(oct->check_db_tq[iq_no].tq);
199 oct->check_db_tq[iq_no].tq = NULL;
200 }
201
202 if (LIO_CN23XX_PF(oct))
203 desc_size =
204 LIO_GET_IQ_INSTR_TYPE_CFG(LIO_CHIP_CONF(oct, cn23xx_pf));
205
206 request_buf = iq->request_list;
207 for (i = 0; i < iq->max_count; i++, request_buf++) {
208 if ((request_buf->reqtype == LIO_REQTYPE_NORESP_NET) ||
209 (request_buf->reqtype == LIO_REQTYPE_NORESP_NET_SG)) {
210 if (request_buf->buf != NULL) {
211 finfo = request_buf->buf;
212 bus_dmamap_sync(iq->txtag, request_buf->map,
213 BUS_DMASYNC_POSTWRITE);
214 bus_dmamap_unload(iq->txtag,
215 request_buf->map);
216 m_freem(finfo->mb);
217 request_buf->buf = NULL;
218 if (request_buf->map != NULL) {
219 bus_dmamap_destroy(iq->txtag,
220 request_buf->map);
221 request_buf->map = NULL;
222 }
223 } else if (request_buf->map != NULL) {
224 bus_dmamap_unload(iq->txtag, request_buf->map);
225 bus_dmamap_destroy(iq->txtag, request_buf->map);
226 request_buf->map = NULL;
227 }
228 }
229 }
230
231 if (iq->br != NULL) {
232 buf_ring_free(iq->br, M_DEVBUF);
233 iq->br = NULL;
234 }
235
236 if (iq->request_list != NULL) {
237 free(iq->request_list, M_DEVBUF);
238 iq->request_list = NULL;
239 }
240
241 if (iq->txtag != NULL) {
242 bus_dma_tag_destroy(iq->txtag);
243 iq->txtag = NULL;
244 }
245
246 if (iq->base_addr) {
247 q_size = iq->max_count * desc_size;
248 lio_dma_free((uint32_t)q_size, iq->base_addr);
249
250 oct->io_qmask.iq &= ~(1ULL << iq_no);
251 bzero(oct->instr_queue[iq_no], sizeof(struct lio_instr_queue));
252 oct->num_iqs--;
253
254 return (0);
255 }
256
257 return (1);
258 }
259
260 /* Return 0 on success, 1 on failure */
261 int
lio_setup_iq(struct octeon_device * oct,int ifidx,int q_index,union octeon_txpciq txpciq,uint32_t num_descs)262 lio_setup_iq(struct octeon_device *oct, int ifidx, int q_index,
263 union octeon_txpciq txpciq, uint32_t num_descs)
264 {
265 uint32_t iq_no = (uint32_t)txpciq.s.q_no;
266
267 if (oct->instr_queue[iq_no]->oct_dev != NULL) {
268 lio_dev_dbg(oct, "IQ is in use. Cannot create the IQ: %d again\n",
269 iq_no);
270 oct->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64;
271 return (0);
272 }
273
274 oct->instr_queue[iq_no]->q_index = q_index;
275 oct->instr_queue[iq_no]->ifidx = ifidx;
276
277 if (lio_init_instr_queue(oct, txpciq, num_descs)) {
278 lio_delete_instr_queue(oct, iq_no);
279 return (1);
280 }
281
282 oct->num_iqs++;
283 if (oct->fn_list.enable_io_queues(oct))
284 return (1);
285
286 return (0);
287 }
288
289 int
lio_wait_for_instr_fetch(struct octeon_device * oct)290 lio_wait_for_instr_fetch(struct octeon_device *oct)
291 {
292 int i, retry = 1000, pending, instr_cnt = 0;
293
294 do {
295 instr_cnt = 0;
296
297 for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) {
298 if (!(oct->io_qmask.iq & BIT_ULL(i)))
299 continue;
300 pending = atomic_load_acq_int(
301 &oct->instr_queue[i]->instr_pending);
302 if (pending)
303 __lio_check_db_timeout(oct, i);
304 instr_cnt += pending;
305 }
306
307 if (instr_cnt == 0)
308 break;
309
310 lio_sleep_timeout(1);
311
312 } while (retry-- && instr_cnt);
313
314 return (instr_cnt);
315 }
316
317 static inline void
lio_ring_doorbell(struct octeon_device * oct,struct lio_instr_queue * iq)318 lio_ring_doorbell(struct octeon_device *oct, struct lio_instr_queue *iq)
319 {
320
321 if (atomic_load_acq_int(&oct->status) == LIO_DEV_RUNNING) {
322 lio_write_csr32(oct, iq->doorbell_reg, iq->fill_cnt);
323 /* make sure doorbell write goes through */
324 __compiler_membar();
325 iq->fill_cnt = 0;
326 iq->last_db_time = ticks;
327 return;
328 }
329 }
330
331 static inline void
__lio_copy_cmd_into_iq(struct lio_instr_queue * iq,uint8_t * cmd)332 __lio_copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd)
333 {
334 uint8_t *iqptr, cmdsize;
335
336 cmdsize = ((iq->iqcmd_64B) ? 64 : 32);
337 iqptr = iq->base_addr + (cmdsize * iq->host_write_index);
338
339 memcpy(iqptr, cmd, cmdsize);
340 }
341
342 static inline struct lio_iq_post_status
__lio_post_command2(struct lio_instr_queue * iq,uint8_t * cmd)343 __lio_post_command2(struct lio_instr_queue *iq, uint8_t *cmd)
344 {
345 struct lio_iq_post_status st;
346
347 st.status = LIO_IQ_SEND_OK;
348
349 /*
350 * This ensures that the read index does not wrap around to the same
351 * position if queue gets full before Octeon could fetch any instr.
352 */
353 if (atomic_load_acq_int(&iq->instr_pending) >=
354 (int32_t)(iq->max_count - 1)) {
355 st.status = LIO_IQ_SEND_FAILED;
356 st.index = -1;
357 return (st);
358 }
359
360 if (atomic_load_acq_int(&iq->instr_pending) >=
361 (int32_t)(iq->max_count - 2))
362 st.status = LIO_IQ_SEND_STOP;
363
364 __lio_copy_cmd_into_iq(iq, cmd);
365
366 /* "index" is returned, host_write_index is modified. */
367 st.index = iq->host_write_index;
368 iq->host_write_index = lio_incr_index(iq->host_write_index, 1,
369 iq->max_count);
370 iq->fill_cnt++;
371
372 /*
373 * Flush the command into memory. We need to be sure the data is in
374 * memory before indicating that the instruction is pending.
375 */
376 wmb();
377
378 atomic_add_int(&iq->instr_pending, 1);
379
380 return (st);
381 }
382
383 static inline void
__lio_add_to_request_list(struct lio_instr_queue * iq,int idx,void * buf,int reqtype)384 __lio_add_to_request_list(struct lio_instr_queue *iq, int idx, void *buf,
385 int reqtype)
386 {
387
388 iq->request_list[idx].buf = buf;
389 iq->request_list[idx].reqtype = reqtype;
390 }
391
392 /* Can only run in process context */
393 int
lio_process_iq_request_list(struct octeon_device * oct,struct lio_instr_queue * iq,uint32_t budget)394 lio_process_iq_request_list(struct octeon_device *oct,
395 struct lio_instr_queue *iq, uint32_t budget)
396 {
397 struct lio_soft_command *sc;
398 struct octeon_instr_irh *irh = NULL;
399 struct lio_mbuf_free_info *finfo;
400 void *buf;
401 uint32_t inst_count = 0;
402 uint32_t old = iq->flush_index;
403 int reqtype;
404
405 while (old != iq->octeon_read_index) {
406 reqtype = iq->request_list[old].reqtype;
407 buf = iq->request_list[old].buf;
408 finfo = buf;
409
410 if (reqtype == LIO_REQTYPE_NONE)
411 goto skip_this;
412
413 switch (reqtype) {
414 case LIO_REQTYPE_NORESP_NET:
415 lio_free_mbuf(iq, buf);
416 break;
417 case LIO_REQTYPE_NORESP_NET_SG:
418 lio_free_sgmbuf(iq, buf);
419 break;
420 case LIO_REQTYPE_RESP_NET:
421 case LIO_REQTYPE_SOFT_COMMAND:
422 sc = buf;
423 if (LIO_CN23XX_PF(oct))
424 irh = (struct octeon_instr_irh *)
425 &sc->cmd.cmd3.irh;
426 if (irh->rflag) {
427 /*
428 * We're expecting a response from Octeon.
429 * It's up to lio_process_ordered_list() to
430 * process sc. Add sc to the ordered soft
431 * command response list because we expect
432 * a response from Octeon.
433 */
434 mtx_lock(&oct->response_list
435 [LIO_ORDERED_SC_LIST].lock);
436 atomic_add_int(&oct->response_list
437 [LIO_ORDERED_SC_LIST].
438 pending_req_count, 1);
439 STAILQ_INSERT_TAIL(&oct->response_list
440 [LIO_ORDERED_SC_LIST].
441 head, &sc->node, entries);
442 mtx_unlock(&oct->response_list
443 [LIO_ORDERED_SC_LIST].lock);
444 } else {
445 if (sc->callback != NULL) {
446 /* This callback must not sleep */
447 sc->callback(oct, LIO_REQUEST_DONE,
448 sc->callback_arg);
449 }
450 }
451
452 break;
453 default:
454 lio_dev_err(oct, "%s Unknown reqtype: %d buf: %p at idx %d\n",
455 __func__, reqtype, buf, old);
456 }
457
458 iq->request_list[old].buf = NULL;
459 iq->request_list[old].reqtype = 0;
460
461 skip_this:
462 inst_count++;
463 old = lio_incr_index(old, 1, iq->max_count);
464
465 if ((budget) && (inst_count >= budget))
466 break;
467 }
468
469 iq->flush_index = old;
470
471 return (inst_count);
472 }
473
474 /* Can only be called from process context */
475 int
lio_flush_iq(struct octeon_device * oct,struct lio_instr_queue * iq,uint32_t budget)476 lio_flush_iq(struct octeon_device *oct, struct lio_instr_queue *iq,
477 uint32_t budget)
478 {
479 uint32_t inst_processed = 0;
480 uint32_t tot_inst_processed = 0;
481 int tx_done = 1;
482
483 if (!mtx_trylock(&iq->iq_flush_running_lock))
484 return (tx_done);
485
486 mtx_lock(&iq->lock);
487
488 iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);
489
490 do {
491 /* Process any outstanding IQ packets. */
492 if (iq->flush_index == iq->octeon_read_index)
493 break;
494
495 if (budget)
496 inst_processed =
497 lio_process_iq_request_list(oct, iq,
498 budget -
499 tot_inst_processed);
500 else
501 inst_processed =
502 lio_process_iq_request_list(oct, iq, 0);
503
504 if (inst_processed) {
505 atomic_subtract_int(&iq->instr_pending, inst_processed);
506 iq->stats.instr_processed += inst_processed;
507 }
508 tot_inst_processed += inst_processed;
509 inst_processed = 0;
510
511 } while (tot_inst_processed < budget);
512
513 if (budget && (tot_inst_processed >= budget))
514 tx_done = 0;
515
516 iq->last_db_time = ticks;
517
518 mtx_unlock(&iq->lock);
519
520 mtx_unlock(&iq->iq_flush_running_lock);
521
522 return (tx_done);
523 }
524
525 /*
526 * Process instruction queue after timeout.
527 * This routine gets called from a taskqueue or when removing the module.
528 */
529 static void
__lio_check_db_timeout(struct octeon_device * oct,uint64_t iq_no)530 __lio_check_db_timeout(struct octeon_device *oct, uint64_t iq_no)
531 {
532 struct lio_instr_queue *iq;
533 uint64_t next_time;
534
535 if (oct == NULL)
536 return;
537
538 iq = oct->instr_queue[iq_no];
539 if (iq == NULL)
540 return;
541
542 if (atomic_load_acq_int(&iq->instr_pending)) {
543 /* If ticks - last_db_time < db_timeout do nothing */
544 next_time = iq->last_db_time + lio_ms_to_ticks(iq->db_timeout);
545 if (!lio_check_timeout(ticks, next_time))
546 return;
547
548 iq->last_db_time = ticks;
549
550 /* Flush the instruction queue */
551 lio_flush_iq(oct, iq, 0);
552
553 lio_enable_irq(NULL, iq);
554 }
555
556 if (oct->props.ifp != NULL && iq->br != NULL) {
557 if (mtx_trylock(&iq->enq_lock)) {
558 if (!drbr_empty(oct->props.ifp, iq->br))
559 lio_mq_start_locked(oct->props.ifp, iq);
560
561 mtx_unlock(&iq->enq_lock);
562 }
563 }
564 }
565
566 /*
567 * Called by the Poll thread at regular intervals to check the instruction
568 * queue for commands to be posted and for commands that were fetched by Octeon.
569 */
570 static void
lio_check_db_timeout(void * arg,int pending)571 lio_check_db_timeout(void *arg, int pending)
572 {
573 struct lio_tq *db_tq = (struct lio_tq *)arg;
574 struct octeon_device *oct = db_tq->ctxptr;
575 uint64_t iq_no = db_tq->ctxul;
576 uint32_t delay = 10;
577
578 __lio_check_db_timeout(oct, iq_no);
579 taskqueue_enqueue_timeout(db_tq->tq, &db_tq->work,
580 lio_ms_to_ticks(delay));
581 }
582
583 int
lio_send_command(struct octeon_device * oct,uint32_t iq_no,uint32_t force_db,void * cmd,void * buf,uint32_t datasize,uint32_t reqtype)584 lio_send_command(struct octeon_device *oct, uint32_t iq_no,
585 uint32_t force_db, void *cmd, void *buf,
586 uint32_t datasize, uint32_t reqtype)
587 {
588 struct lio_iq_post_status st;
589 struct lio_instr_queue *iq = oct->instr_queue[iq_no];
590
591 /*
592 * Get the lock and prevent other tasks and tx interrupt handler
593 * from running.
594 */
595 mtx_lock(&iq->post_lock);
596
597 st = __lio_post_command2(iq, cmd);
598
599 if (st.status != LIO_IQ_SEND_FAILED) {
600 __lio_add_to_request_list(iq, st.index, buf, reqtype);
601 LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize);
602 LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1);
603
604 if (force_db || (st.status == LIO_IQ_SEND_STOP))
605 lio_ring_doorbell(oct, iq);
606 } else {
607 LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);
608 }
609
610 mtx_unlock(&iq->post_lock);
611
612 /*
613 * This is only done here to expedite packets being flushed for
614 * cases where there are no IQ completion interrupts.
615 */
616
617 return (st.status);
618 }
619
620 void
lio_prepare_soft_command(struct octeon_device * oct,struct lio_soft_command * sc,uint8_t opcode,uint8_t subcode,uint32_t irh_ossp,uint64_t ossp0,uint64_t ossp1)621 lio_prepare_soft_command(struct octeon_device *oct, struct lio_soft_command *sc,
622 uint8_t opcode, uint8_t subcode, uint32_t irh_ossp,
623 uint64_t ossp0, uint64_t ossp1)
624 {
625 struct lio_config *lio_cfg;
626 struct octeon_instr_ih3 *ih3;
627 struct octeon_instr_pki_ih3 *pki_ih3;
628 struct octeon_instr_irh *irh;
629 struct octeon_instr_rdp *rdp;
630
631 KASSERT(opcode <= 15, ("%s, %d, opcode > 15", __func__, __LINE__));
632 KASSERT(subcode <= 127, ("%s, %d, opcode > 127", __func__, __LINE__));
633
634 lio_cfg = lio_get_conf(oct);
635
636 if (LIO_CN23XX_PF(oct)) {
637 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
638
639 ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind;
640
641 pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3;
642
643 pki_ih3->w = 1;
644 pki_ih3->raw = 1;
645 pki_ih3->utag = 1;
646 pki_ih3->uqpg = oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg;
647 pki_ih3->utt = 1;
648 pki_ih3->tag = LIO_CONTROL;
649 pki_ih3->tagtype = LIO_ATOMIC_TAG;
650 pki_ih3->qpg = oct->instr_queue[sc->iq_no]->txpciq.s.qpg;
651 pki_ih3->pm = 0x7;
652 pki_ih3->sl = 8;
653
654 if (sc->datasize)
655 ih3->dlengsz = sc->datasize;
656
657 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
658 irh->opcode = opcode;
659 irh->subcode = subcode;
660
661 /* opcode/subcode specific parameters (ossp) */
662 irh->ossp = irh_ossp;
663 sc->cmd.cmd3.ossp[0] = ossp0;
664 sc->cmd.cmd3.ossp[1] = ossp1;
665
666 if (sc->rdatasize) {
667 rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
668 rdp->pcie_port = oct->pcie_port;
669 rdp->rlen = sc->rdatasize;
670
671 irh->rflag = 1;
672 /* PKI IH3 */
673 /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */
674 ih3->fsz = LIO_SOFTCMDRESP_IH3;
675 } else {
676 irh->rflag = 0;
677 /* PKI IH3 */
678 /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */
679 ih3->fsz = LIO_PCICMD_O3;
680 }
681 }
682 }
683
684 int
lio_send_soft_command(struct octeon_device * oct,struct lio_soft_command * sc)685 lio_send_soft_command(struct octeon_device *oct, struct lio_soft_command *sc)
686 {
687 struct octeon_instr_ih3 *ih3;
688 struct octeon_instr_irh *irh;
689 uint32_t len = 0;
690
691 if (LIO_CN23XX_PF(oct)) {
692 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
693 if (ih3->dlengsz) {
694 KASSERT(sc->dmadptr, ("%s, %d, sc->dmadptr is NULL",
695 __func__, __LINE__));
696 sc->cmd.cmd3.dptr = sc->dmadptr;
697 }
698
699 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
700 if (irh->rflag) {
701 KASSERT(sc->dmarptr, ("%s, %d, sc->dmarptr is NULL",
702 __func__, __LINE__));
703 KASSERT(sc->status_word, ("%s, %d, sc->status_word is NULL",
704 __func__, __LINE__));
705 *sc->status_word = COMPLETION_WORD_INIT;
706 sc->cmd.cmd3.rptr = sc->dmarptr;
707 }
708 len = (uint32_t)ih3->dlengsz;
709 }
710 if (sc->wait_time)
711 sc->timeout = ticks + lio_ms_to_ticks(sc->wait_time);
712
713 return (lio_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,
714 len, LIO_REQTYPE_SOFT_COMMAND));
715 }
716
717 int
lio_setup_sc_buffer_pool(struct octeon_device * oct)718 lio_setup_sc_buffer_pool(struct octeon_device *oct)
719 {
720 struct lio_soft_command *sc;
721 uint64_t dma_addr;
722 int i;
723
724 STAILQ_INIT(&oct->sc_buf_pool.head);
725 mtx_init(&oct->sc_buf_pool.lock, "sc_pool_lock", NULL, MTX_DEF);
726 atomic_store_rel_int(&oct->sc_buf_pool.alloc_buf_count, 0);
727
728 for (i = 0; i < LIO_MAX_SOFT_COMMAND_BUFFERS; i++) {
729 sc = (struct lio_soft_command *)
730 lio_dma_alloc(LIO_SOFT_COMMAND_BUFFER_SIZE, (vm_paddr_t *)&dma_addr);
731 if (sc == NULL) {
732 lio_free_sc_buffer_pool(oct);
733 return (1);
734 }
735
736 sc->dma_addr = dma_addr;
737 sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE;
738
739 STAILQ_INSERT_TAIL(&oct->sc_buf_pool.head, &sc->node, entries);
740 }
741
742 return (0);
743 }
744
745 int
lio_free_sc_buffer_pool(struct octeon_device * oct)746 lio_free_sc_buffer_pool(struct octeon_device *oct)
747 {
748 struct lio_stailq_node *tmp, *tmp2;
749 struct lio_soft_command *sc;
750
751 mtx_lock(&oct->sc_buf_pool.lock);
752
753 STAILQ_FOREACH_SAFE(tmp, &oct->sc_buf_pool.head, entries, tmp2) {
754 sc = LIO_STAILQ_FIRST_ENTRY(&oct->sc_buf_pool.head,
755 struct lio_soft_command, node);
756
757 STAILQ_REMOVE_HEAD(&oct->sc_buf_pool.head, entries);
758
759 lio_dma_free(sc->size, sc);
760 }
761
762 STAILQ_INIT(&oct->sc_buf_pool.head);
763
764 mtx_unlock(&oct->sc_buf_pool.lock);
765
766 return (0);
767 }
768
769 struct lio_soft_command *
lio_alloc_soft_command(struct octeon_device * oct,uint32_t datasize,uint32_t rdatasize,uint32_t ctxsize)770 lio_alloc_soft_command(struct octeon_device *oct, uint32_t datasize,
771 uint32_t rdatasize, uint32_t ctxsize)
772 {
773 struct lio_soft_command *sc = NULL;
774 struct lio_stailq_node *tmp;
775 uint64_t dma_addr;
776 uint32_t size;
777 uint32_t offset = sizeof(struct lio_soft_command);
778
779 KASSERT((offset + datasize + rdatasize + ctxsize) <=
780 LIO_SOFT_COMMAND_BUFFER_SIZE,
781 ("%s, %d, offset + datasize + rdatasize + ctxsize > LIO_SOFT_COMMAND_BUFFER_SIZE",
782 __func__, __LINE__));
783
784 mtx_lock(&oct->sc_buf_pool.lock);
785
786 if (STAILQ_EMPTY(&oct->sc_buf_pool.head)) {
787 mtx_unlock(&oct->sc_buf_pool.lock);
788 return (NULL);
789 }
790 tmp = STAILQ_LAST(&oct->sc_buf_pool.head, lio_stailq_node, entries);
791
792 STAILQ_REMOVE(&oct->sc_buf_pool.head, tmp, lio_stailq_node, entries);
793
794 atomic_add_int(&oct->sc_buf_pool.alloc_buf_count, 1);
795
796 mtx_unlock(&oct->sc_buf_pool.lock);
797
798 sc = (struct lio_soft_command *)tmp;
799
800 dma_addr = sc->dma_addr;
801 size = sc->size;
802
803 bzero(sc, sc->size);
804
805 sc->dma_addr = dma_addr;
806 sc->size = size;
807
808 if (ctxsize) {
809 sc->ctxptr = (uint8_t *)sc + offset;
810 sc->ctxsize = ctxsize;
811 }
812
813 /* Start data at 128 byte boundary */
814 offset = (offset + ctxsize + 127) & 0xffffff80;
815
816 if (datasize) {
817 sc->virtdptr = (uint8_t *)sc + offset;
818 sc->dmadptr = dma_addr + offset;
819 sc->datasize = datasize;
820 }
821 /* Start rdata at 128 byte boundary */
822 offset = (offset + datasize + 127) & 0xffffff80;
823
824 if (rdatasize) {
825 KASSERT(rdatasize >= 16, ("%s, %d, rdatasize < 16", __func__,
826 __LINE__));
827 sc->virtrptr = (uint8_t *)sc + offset;
828 sc->dmarptr = dma_addr + offset;
829 sc->rdatasize = rdatasize;
830 sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) +
831 rdatasize - 8);
832 }
833 return (sc);
834 }
835
836 void
lio_free_soft_command(struct octeon_device * oct,struct lio_soft_command * sc)837 lio_free_soft_command(struct octeon_device *oct,
838 struct lio_soft_command *sc)
839 {
840
841 mtx_lock(&oct->sc_buf_pool.lock);
842
843 STAILQ_INSERT_TAIL(&oct->sc_buf_pool.head, &sc->node, entries);
844
845 atomic_subtract_int(&oct->sc_buf_pool.alloc_buf_count, 1);
846
847 mtx_unlock(&oct->sc_buf_pool.lock);
848 }
849