1 /*-
2 * Copyright (c) 2016-2020 Mellanox Technologies. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "opt_rss.h"
27 #include "opt_ratelimit.h"
28
29 #include <dev/mlx5/mlx5_en/en.h>
30
31 #ifdef RATELIMIT
32
33 static int mlx5e_rl_open_workers(struct mlx5e_priv *);
34 static void mlx5e_rl_close_workers(struct mlx5e_priv *);
35 static int mlx5e_rl_sysctl_show_rate_table(SYSCTL_HANDLER_ARGS);
36 static void mlx5e_rl_sysctl_add_u64_oid(struct mlx5e_rl_priv_data *, unsigned x,
37 struct sysctl_oid *, const char *name, const char *desc);
38 static void mlx5e_rl_sysctl_add_stats_u64_oid(struct mlx5e_rl_priv_data *rl, unsigned x,
39 struct sysctl_oid *node, const char *name, const char *desc);
40 static int mlx5e_rl_tx_limit_add(struct mlx5e_rl_priv_data *, uint64_t value);
41 static int mlx5e_rl_tx_limit_clr(struct mlx5e_rl_priv_data *, uint64_t value);
42
43 static void
mlx5e_rl_build_sq_param(struct mlx5e_rl_priv_data * rl,struct mlx5e_sq_param * param)44 mlx5e_rl_build_sq_param(struct mlx5e_rl_priv_data *rl,
45 struct mlx5e_sq_param *param)
46 {
47 void *sqc = param->sqc;
48 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
49 uint8_t log_sq_size = order_base_2(rl->param.tx_queue_size);
50
51 MLX5_SET(wq, wq, log_wq_sz, log_sq_size);
52 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
53 MLX5_SET(wq, wq, pd, rl->priv->pdn);
54
55 param->wq.linear = 1;
56 }
57
58 static void
mlx5e_rl_build_cq_param(struct mlx5e_rl_priv_data * rl,struct mlx5e_cq_param * param)59 mlx5e_rl_build_cq_param(struct mlx5e_rl_priv_data *rl,
60 struct mlx5e_cq_param *param)
61 {
62 void *cqc = param->cqc;
63 uint8_t log_sq_size = order_base_2(rl->param.tx_queue_size);
64
65 MLX5_SET(cqc, cqc, log_cq_size, log_sq_size);
66 MLX5_SET(cqc, cqc, cq_period, rl->param.tx_coalesce_usecs);
67 MLX5_SET(cqc, cqc, cq_max_count, rl->param.tx_coalesce_pkts);
68 MLX5_SET(cqc, cqc, uar_page, rl->priv->mdev->priv.uar->index);
69
70 switch (rl->param.tx_coalesce_mode) {
71 case 0:
72 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
73 break;
74 default:
75 if (MLX5_CAP_GEN(rl->priv->mdev, cq_period_start_from_cqe))
76 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
77 else
78 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
79 break;
80 }
81 }
82
83 static void
mlx5e_rl_build_channel_param(struct mlx5e_rl_priv_data * rl,struct mlx5e_rl_channel_param * cparam)84 mlx5e_rl_build_channel_param(struct mlx5e_rl_priv_data *rl,
85 struct mlx5e_rl_channel_param *cparam)
86 {
87 memset(cparam, 0, sizeof(*cparam));
88
89 mlx5e_rl_build_sq_param(rl, &cparam->sq);
90 mlx5e_rl_build_cq_param(rl, &cparam->cq);
91 }
92
93 static int
mlx5e_rl_create_sq(struct mlx5e_priv * priv,struct mlx5e_sq * sq,struct mlx5e_sq_param * param,int ix)94 mlx5e_rl_create_sq(struct mlx5e_priv *priv, struct mlx5e_sq *sq,
95 struct mlx5e_sq_param *param, int ix)
96 {
97 struct mlx5_core_dev *mdev = priv->mdev;
98 void *sqc = param->sqc;
99 void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
100 int err;
101
102 /* Create DMA descriptor TAG */
103 if ((err = -bus_dma_tag_create(
104 bus_get_dma_tag(mdev->pdev->dev.bsddev),
105 1, /* any alignment */
106 0, /* no boundary */
107 BUS_SPACE_MAXADDR, /* lowaddr */
108 BUS_SPACE_MAXADDR, /* highaddr */
109 NULL, NULL, /* filter, filterarg */
110 MLX5E_MAX_TX_PAYLOAD_SIZE, /* maxsize */
111 MLX5E_MAX_TX_MBUF_FRAGS, /* nsegments */
112 MLX5E_MAX_TX_MBUF_SIZE, /* maxsegsize */
113 0, /* flags */
114 NULL, NULL, /* lockfunc, lockfuncarg */
115 &sq->dma_tag)))
116 goto done;
117
118 sq->mkey_be = cpu_to_be32(priv->mr.key);
119 sq->ifp = priv->ifp;
120 sq->priv = priv;
121
122 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, &sq->wq,
123 &sq->wq_ctrl);
124 if (err)
125 goto err_free_dma_tag;
126
127 sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
128
129 err = mlx5e_alloc_sq_db(sq);
130 if (err)
131 goto err_sq_wq_destroy;
132
133 mlx5e_update_sq_inline(sq);
134
135 return (0);
136
137 err_sq_wq_destroy:
138 mlx5_wq_destroy(&sq->wq_ctrl);
139 err_free_dma_tag:
140 bus_dma_tag_destroy(sq->dma_tag);
141 done:
142 return (err);
143 }
144
145 static void
mlx5e_rl_destroy_sq(struct mlx5e_sq * sq)146 mlx5e_rl_destroy_sq(struct mlx5e_sq *sq)
147 {
148
149 mlx5e_free_sq_db(sq);
150 mlx5_wq_destroy(&sq->wq_ctrl);
151 bus_dma_tag_destroy(sq->dma_tag);
152 }
153
154 static int
mlx5e_rl_query_sq(struct mlx5e_sq * sq)155 mlx5e_rl_query_sq(struct mlx5e_sq *sq)
156 {
157 void *out;
158 int inlen;
159 int err;
160
161 inlen = MLX5_ST_SZ_BYTES(query_sq_out);
162 out = mlx5_vzalloc(inlen);
163 if (!out)
164 return -ENOMEM;
165
166 err = mlx5_core_query_sq(sq->priv->mdev, sq->sqn, out);
167 if (err)
168 goto out;
169
170 sq->queue_handle = MLX5_GET(query_sq_out, out, sq_context.queue_handle);
171
172 out:
173 kvfree(out);
174 return err;
175 }
176
177 static int
mlx5e_rl_open_sq(struct mlx5e_priv * priv,struct mlx5e_sq * sq,struct mlx5e_sq_param * param,int ix)178 mlx5e_rl_open_sq(struct mlx5e_priv *priv, struct mlx5e_sq *sq,
179 struct mlx5e_sq_param *param, int ix)
180 {
181 int err;
182
183 err = mlx5e_rl_create_sq(priv, sq, param, ix);
184 if (err)
185 return (err);
186
187 err = mlx5e_enable_sq(sq, param, &priv->channel[ix].bfreg, priv->rl.tisn);
188 if (err)
189 goto err_destroy_sq;
190
191 err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
192 if (err)
193 goto err_disable_sq;
194
195 if (MLX5_CAP_QOS(priv->mdev, qos_remap_pp)) {
196 err = mlx5e_rl_query_sq(sq);
197 if (err) {
198 mlx5_en_err(priv->ifp, "Failed retrieving send queue handle for"
199 "SQ remap - sqn=%u, err=(%d)\n", sq->sqn, err);
200 sq->queue_handle = MLX5_INVALID_QUEUE_HANDLE;
201 }
202 } else
203 sq->queue_handle = MLX5_INVALID_QUEUE_HANDLE;
204
205 WRITE_ONCE(sq->running, 1);
206
207 return (0);
208
209 err_disable_sq:
210 mlx5e_disable_sq(sq);
211 err_destroy_sq:
212 mlx5e_rl_destroy_sq(sq);
213
214 return (err);
215 }
216
217 static void
mlx5e_rl_chan_mtx_init(struct mlx5e_priv * priv,struct mlx5e_sq * sq)218 mlx5e_rl_chan_mtx_init(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
219 {
220 mtx_init(&sq->lock, "mlx5tx-rl", NULL, MTX_DEF);
221 mtx_init(&sq->comp_lock, "mlx5comp-rl", NULL, MTX_DEF);
222
223 callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
224
225 sq->cev_factor = priv->rl.param.tx_completion_fact;
226
227 /* ensure the TX completion event factor is not zero */
228 if (sq->cev_factor == 0)
229 sq->cev_factor = 1;
230 }
231
232 static int
mlx5e_rl_open_channel(struct mlx5e_rl_worker * rlw,int eq_ix,struct mlx5e_rl_channel_param * cparam,struct mlx5e_sq * volatile * ppsq)233 mlx5e_rl_open_channel(struct mlx5e_rl_worker *rlw, int eq_ix,
234 struct mlx5e_rl_channel_param *cparam,
235 struct mlx5e_sq *volatile *ppsq)
236 {
237 struct mlx5e_priv *priv = rlw->priv;
238 struct mlx5e_sq *sq;
239 int err;
240
241 sq = malloc(sizeof(*sq), M_MLX5EN, M_WAITOK | M_ZERO);
242
243 /* init mutexes */
244 mlx5e_rl_chan_mtx_init(priv, sq);
245
246 /* open TX completion queue */
247 err = mlx5e_open_cq(priv, &cparam->cq, &sq->cq,
248 &mlx5e_tx_cq_comp, eq_ix);
249 if (err)
250 goto err_free;
251
252 err = mlx5e_rl_open_sq(priv, sq, &cparam->sq, eq_ix);
253 if (err)
254 goto err_close_tx_cq;
255
256 /* store TX channel pointer */
257 *ppsq = sq;
258
259 /* poll TX queue initially */
260 sq->cq.mcq.comp(&sq->cq.mcq, NULL);
261
262 return (0);
263
264 err_close_tx_cq:
265 mlx5e_close_cq(&sq->cq);
266
267 err_free:
268 /* destroy mutexes */
269 mtx_destroy(&sq->lock);
270 mtx_destroy(&sq->comp_lock);
271 free(sq, M_MLX5EN);
272 atomic_add_64(&priv->rl.stats.tx_allocate_resource_failure, 1ULL);
273 return (err);
274 }
275
276 static void
mlx5e_rl_close_channel(struct mlx5e_sq * volatile * ppsq)277 mlx5e_rl_close_channel(struct mlx5e_sq *volatile *ppsq)
278 {
279 struct mlx5e_sq *sq = *ppsq;
280
281 /* check if channel is already closed */
282 if (sq == NULL)
283 return;
284 /* ensure channel pointer is no longer used */
285 *ppsq = NULL;
286
287 /* teardown and destroy SQ */
288 mlx5e_drain_sq(sq);
289 mlx5e_disable_sq(sq);
290 mlx5e_rl_destroy_sq(sq);
291
292 /* close CQ */
293 mlx5e_close_cq(&sq->cq);
294
295 /* destroy mutexes */
296 mtx_destroy(&sq->lock);
297 mtx_destroy(&sq->comp_lock);
298
299 free(sq, M_MLX5EN);
300 }
301
302 static void
mlx5e_rl_sync_tx_completion_fact(struct mlx5e_rl_priv_data * rl)303 mlx5e_rl_sync_tx_completion_fact(struct mlx5e_rl_priv_data *rl)
304 {
305 /*
306 * Limit the maximum distance between completion events to
307 * half of the currently set TX queue size.
308 *
309 * The maximum number of queue entries a single IP packet can
310 * consume is given by MLX5_SEND_WQE_MAX_WQEBBS.
311 *
312 * The worst case max value is then given as below:
313 */
314 uint64_t max = rl->param.tx_queue_size /
315 (2 * MLX5_SEND_WQE_MAX_WQEBBS);
316
317 /*
318 * Update the maximum completion factor value in case the
319 * tx_queue_size field changed. Ensure we don't overflow
320 * 16-bits.
321 */
322 if (max < 1)
323 max = 1;
324 else if (max > 65535)
325 max = 65535;
326 rl->param.tx_completion_fact_max = max;
327
328 /*
329 * Verify that the current TX completion factor is within the
330 * given limits:
331 */
332 if (rl->param.tx_completion_fact < 1)
333 rl->param.tx_completion_fact = 1;
334 else if (rl->param.tx_completion_fact > max)
335 rl->param.tx_completion_fact = max;
336 }
337
338 static int
mlx5e_rl_modify_sq(struct mlx5e_sq * sq,uint16_t rl_index)339 mlx5e_rl_modify_sq(struct mlx5e_sq *sq, uint16_t rl_index)
340 {
341 struct mlx5e_priv *priv = sq->priv;
342 struct mlx5_core_dev *mdev = priv->mdev;
343
344 void *in;
345 void *sqc;
346 int inlen;
347 int err;
348
349 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
350 in = mlx5_vzalloc(inlen);
351 if (in == NULL)
352 return (-ENOMEM);
353
354 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
355
356 MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
357 MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RDY);
358 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
359 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY);
360 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
361
362 err = mlx5_core_modify_sq(mdev, in, inlen);
363
364 kvfree(in);
365
366 return (err);
367 }
368
369 /*
370 * This function will search the configured rate limit table for the
371 * best match to avoid that a single socket based application can
372 * allocate all the available hardware rates. If the user selected
373 * rate deviates too much from the closes rate available in the rate
374 * limit table, unlimited rate will be selected.
375 */
376 static uint64_t
mlx5e_rl_find_best_rate_locked(struct mlx5e_rl_priv_data * rl,uint64_t user_rate)377 mlx5e_rl_find_best_rate_locked(struct mlx5e_rl_priv_data *rl, uint64_t user_rate)
378 {
379 uint64_t distance = -1ULL;
380 uint64_t diff;
381 uint64_t retval = 0; /* unlimited */
382 uint64_t x;
383
384 /* search for closest rate */
385 for (x = 0; x != rl->param.tx_rates_def; x++) {
386 uint64_t rate = rl->rate_limit_table[x];
387 if (rate == 0)
388 continue;
389
390 if (rate > user_rate)
391 diff = rate - user_rate;
392 else
393 diff = user_rate - rate;
394
395 /* check if distance is smaller than previous rate */
396 if (diff < distance) {
397 distance = diff;
398 retval = rate;
399 }
400 }
401
402 /* range check for multiplication below */
403 if (user_rate > rl->param.tx_limit_max)
404 user_rate = rl->param.tx_limit_max;
405
406 /* fallback to unlimited, if rate deviates too much */
407 if (distance > howmany(user_rate *
408 rl->param.tx_allowed_deviation, 1000ULL))
409 retval = 0;
410
411 return (retval);
412 }
413
414 static int
mlx5e_rl_post_sq_remap_wqe(struct mlx5e_iq * iq,u32 scq_handle,u32 sq_handle,struct mlx5e_rl_channel * sq_channel)415 mlx5e_rl_post_sq_remap_wqe(struct mlx5e_iq *iq, u32 scq_handle, u32 sq_handle,
416 struct mlx5e_rl_channel *sq_channel)
417 {
418 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_qos_remap_wqe),
419 MLX5_SEND_WQE_DS);
420 struct mlx5e_tx_qos_remap_wqe *wqe;
421 int pi;
422
423 mtx_lock(&iq->lock);
424 pi = mlx5e_iq_get_producer_index(iq);
425 if (pi < 0) {
426 mtx_unlock(&iq->lock);
427 return (-ENOMEM);
428 }
429 wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
430
431 memset(wqe, 0, sizeof(*wqe));
432
433 wqe->qos_remap.qos_handle = cpu_to_be32(scq_handle);
434 wqe->qos_remap.queue_handle = cpu_to_be32(sq_handle);
435
436 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
437 MLX5_OPCODE_QOS_REMAP);
438 wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
439 wqe->ctrl.imm = cpu_to_be32(iq->priv->tisn[0] << 8);
440 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
441
442 /* copy data for doorbell */
443 memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
444
445 iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
446 iq->data[pi].p_refcount = &sq_channel->refcount;
447 atomic_add_int(iq->data[pi].p_refcount, 1);
448 iq->pc += iq->data[pi].num_wqebbs;
449
450 mlx5e_iq_notify_hw(iq);
451
452 mtx_unlock(&iq->lock);
453
454 return (0); /* success */
455 }
456
457 static int
mlx5e_rl_remap_sq(struct mlx5e_sq * sq,uint16_t index,struct mlx5e_rl_channel * sq_channel)458 mlx5e_rl_remap_sq(struct mlx5e_sq *sq, uint16_t index,
459 struct mlx5e_rl_channel *sq_channel)
460 {
461 struct mlx5e_channel *iq_channel;
462 u32 scq_handle;
463 u32 sq_handle;
464 int error;
465
466 /* Specific SQ remap operations should be handled by same IQ */
467 iq_channel = &sq->priv->channel[sq->sqn % sq->priv->params.num_channels];
468
469 sq_handle = sq->queue_handle;
470 scq_handle = mlx5_rl_get_scq_handle(sq->priv->mdev, index);
471
472 if (sq_handle == MLX5_INVALID_QUEUE_HANDLE ||
473 scq_handle == MLX5_INVALID_QUEUE_HANDLE)
474 error = -1;
475 else
476 error = mlx5e_rl_post_sq_remap_wqe(&iq_channel->iq, scq_handle,
477 sq_handle, sq_channel);
478
479 return (error);
480 }
481
482 /*
483 * This function sets the requested rate for a rate limit channel, in
484 * bits per second. The requested rate will be filtered through the
485 * find best rate function above.
486 */
487 static int
mlx5e_rlw_channel_set_rate_locked(struct mlx5e_rl_worker * rlw,struct mlx5e_rl_channel * channel,uint64_t rate)488 mlx5e_rlw_channel_set_rate_locked(struct mlx5e_rl_worker *rlw,
489 struct mlx5e_rl_channel *channel, uint64_t rate)
490 {
491 struct mlx5e_rl_priv_data *rl = &rlw->priv->rl;
492 struct mlx5e_sq *sq;
493 uint64_t temp;
494 uint16_t index;
495 uint16_t burst;
496 int error;
497 bool use_sq_remap;
498
499 if (rate != 0) {
500 MLX5E_RL_WORKER_UNLOCK(rlw);
501
502 MLX5E_RL_RLOCK(rl);
503
504 /* get current burst size in bytes */
505 temp = rl->param.tx_burst_size *
506 MLX5E_SW2HW_MTU(rlw->priv->ifp->if_mtu);
507
508 /* limit burst size to 64K currently */
509 if (temp > 65535)
510 temp = 65535;
511 burst = temp;
512
513 /* find best rate */
514 rate = mlx5e_rl_find_best_rate_locked(rl, rate);
515
516 MLX5E_RL_RUNLOCK(rl);
517
518 if (rate == 0) {
519 /* rate doesn't exist, fallback to unlimited */
520 index = 0;
521 rate = 0;
522 atomic_add_64(&rlw->priv->rl.stats.tx_modify_rate_failure, 1ULL);
523 } else {
524 /* get a reference on the new rate */
525 error = -mlx5_rl_add_rate(rlw->priv->mdev,
526 howmany(rate, 1000), burst, &index);
527
528 if (error != 0) {
529 /* adding rate failed, fallback to unlimited */
530 index = 0;
531 rate = 0;
532 atomic_add_64(&rlw->priv->rl.stats.tx_add_new_rate_failure, 1ULL);
533 }
534 }
535 MLX5E_RL_WORKER_LOCK(rlw);
536 } else {
537 index = 0;
538 burst = 0; /* default */
539 }
540
541 /* paced <--> non-paced transitions must go via FW */
542 use_sq_remap = MLX5_CAP_QOS(rlw->priv->mdev, qos_remap_pp) &&
543 channel->last_rate != 0 && rate != 0;
544
545 /* atomically swap rates */
546 temp = channel->last_rate;
547 channel->last_rate = rate;
548 rate = temp;
549
550 /* atomically swap burst size */
551 temp = channel->last_burst;
552 channel->last_burst = burst;
553 burst = temp;
554
555 MLX5E_RL_WORKER_UNLOCK(rlw);
556 /* put reference on the old rate, if any */
557 if (rate != 0) {
558 mlx5_rl_remove_rate(rlw->priv->mdev,
559 howmany(rate, 1000), burst);
560 }
561
562 /* set new rate, if SQ is running */
563 sq = channel->sq;
564 if (sq != NULL && READ_ONCE(sq->running) != 0) {
565 if (!use_sq_remap || mlx5e_rl_remap_sq(sq, index, channel)) {
566 while (atomic_load_int(&channel->refcount) != 0 &&
567 rlw->priv->mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR &&
568 pci_channel_offline(rlw->priv->mdev->pdev) == 0)
569 pause("W", 1);
570 error = mlx5e_rl_modify_sq(sq, index);
571 if (error != 0)
572 atomic_add_64(&rlw->priv->rl.stats.tx_modify_rate_failure, 1ULL);
573 }
574 } else
575 error = 0;
576
577 MLX5E_RL_WORKER_LOCK(rlw);
578
579 return (-error);
580 }
581
582 static void
mlx5e_rl_worker(void * arg)583 mlx5e_rl_worker(void *arg)
584 {
585 struct thread *td;
586 struct mlx5e_rl_worker *rlw = arg;
587 struct mlx5e_rl_channel *channel;
588 struct mlx5e_priv *priv;
589 unsigned ix;
590 uint64_t x;
591 int error;
592
593 /* set thread priority */
594 td = curthread;
595
596 thread_lock(td);
597 sched_prio(td, PI_SWI(SWI_NET));
598 thread_unlock(td);
599
600 priv = rlw->priv;
601
602 /* compute completion vector */
603 ix = (rlw - priv->rl.workers) %
604 priv->mdev->priv.eq_table.num_comp_vectors;
605
606 /* TODO bind to CPU */
607
608 /* open all the SQs */
609 MLX5E_RL_WORKER_LOCK(rlw);
610 for (x = 0; x < priv->rl.param.tx_channels_per_worker_def; x++) {
611 struct mlx5e_rl_channel *channel = rlw->channels + x;
612
613 #if !defined(HAVE_RL_PRE_ALLOCATE_CHANNELS)
614 if (channel->state == MLX5E_RL_ST_FREE)
615 continue;
616 #endif
617 MLX5E_RL_WORKER_UNLOCK(rlw);
618
619 MLX5E_RL_RLOCK(&priv->rl);
620 error = mlx5e_rl_open_channel(rlw, ix,
621 &priv->rl.chan_param, &channel->sq);
622 MLX5E_RL_RUNLOCK(&priv->rl);
623
624 MLX5E_RL_WORKER_LOCK(rlw);
625 if (error != 0) {
626 mlx5_en_err(priv->ifp,
627 "mlx5e_rl_open_channel failed: %d\n", error);
628 break;
629 }
630 mlx5e_rlw_channel_set_rate_locked(rlw, channel, channel->init_rate);
631 }
632 while (1) {
633 if (STAILQ_FIRST(&rlw->process_head) == NULL) {
634 /* check if we are tearing down */
635 if (rlw->worker_done != 0)
636 break;
637 cv_wait(&rlw->cv, &rlw->mtx);
638 }
639 /* check if we are tearing down */
640 if (rlw->worker_done != 0)
641 break;
642 channel = STAILQ_FIRST(&rlw->process_head);
643 if (channel != NULL) {
644 STAILQ_REMOVE_HEAD(&rlw->process_head, entry);
645
646 switch (channel->state) {
647 case MLX5E_RL_ST_MODIFY:
648 channel->state = MLX5E_RL_ST_USED;
649 MLX5E_RL_WORKER_UNLOCK(rlw);
650
651 /* create channel by demand */
652 if (channel->sq == NULL) {
653 MLX5E_RL_RLOCK(&priv->rl);
654 error = mlx5e_rl_open_channel(rlw, ix,
655 &priv->rl.chan_param, &channel->sq);
656 MLX5E_RL_RUNLOCK(&priv->rl);
657
658 if (error != 0) {
659 mlx5_en_err(priv->ifp,
660 "mlx5e_rl_open_channel failed: %d\n", error);
661 } else {
662 atomic_add_64(&rlw->priv->rl.stats.tx_open_queues, 1ULL);
663 }
664 } else {
665 mlx5e_resume_sq(channel->sq);
666 }
667
668 MLX5E_RL_WORKER_LOCK(rlw);
669 /* convert from bytes/s to bits/s and set new rate */
670 error = mlx5e_rlw_channel_set_rate_locked(rlw, channel,
671 channel->new_rate * 8ULL);
672 if (error != 0) {
673 mlx5_en_err(priv->ifp,
674 "mlx5e_rlw_channel_set_rate_locked failed: %d\n",
675 error);
676 }
677 break;
678
679 case MLX5E_RL_ST_DESTROY:
680 error = mlx5e_rlw_channel_set_rate_locked(rlw, channel, 0);
681 if (error != 0) {
682 mlx5_en_err(priv->ifp,
683 "mlx5e_rlw_channel_set_rate_locked failed: %d\n",
684 error);
685 }
686 if (channel->sq != NULL) {
687 /*
688 * Make sure all packets are
689 * transmitted before SQ is
690 * returned to free list:
691 */
692 MLX5E_RL_WORKER_UNLOCK(rlw);
693 mlx5e_drain_sq(channel->sq);
694 MLX5E_RL_WORKER_LOCK(rlw);
695 }
696 /* put the channel back into the free list */
697 STAILQ_INSERT_HEAD(&rlw->index_list_head, channel, entry);
698 channel->state = MLX5E_RL_ST_FREE;
699 atomic_add_64(&priv->rl.stats.tx_active_connections, -1ULL);
700 break;
701 default:
702 /* NOP */
703 break;
704 }
705 }
706 }
707
708 /* close all the SQs */
709 for (x = 0; x < priv->rl.param.tx_channels_per_worker_def; x++) {
710 struct mlx5e_rl_channel *channel = rlw->channels + x;
711
712 /* update the initial rate */
713 channel->init_rate = channel->last_rate;
714
715 /* make sure we free up the rate resource */
716 mlx5e_rlw_channel_set_rate_locked(rlw, channel, 0);
717
718 if (channel->sq != NULL) {
719 MLX5E_RL_WORKER_UNLOCK(rlw);
720 mlx5e_rl_close_channel(&channel->sq);
721 atomic_add_64(&rlw->priv->rl.stats.tx_open_queues, -1ULL);
722 MLX5E_RL_WORKER_LOCK(rlw);
723 }
724 }
725
726 rlw->worker_done = 0;
727 cv_broadcast(&rlw->cv);
728 MLX5E_RL_WORKER_UNLOCK(rlw);
729
730 kthread_exit();
731 }
732
733 static int
mlx5e_rl_open_tis(struct mlx5e_priv * priv)734 mlx5e_rl_open_tis(struct mlx5e_priv *priv)
735 {
736 struct mlx5_core_dev *mdev = priv->mdev;
737 u32 in[MLX5_ST_SZ_DW(create_tis_in)];
738 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
739
740 memset(in, 0, sizeof(in));
741
742 MLX5_SET(tisc, tisc, prio, 0);
743 MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
744
745 return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->rl.tisn));
746 }
747
748 static void
mlx5e_rl_close_tis(struct mlx5e_priv * priv)749 mlx5e_rl_close_tis(struct mlx5e_priv *priv)
750 {
751 mlx5_core_destroy_tis(priv->mdev, priv->rl.tisn);
752 }
753
754 static void
mlx5e_rl_set_default_params(struct mlx5e_rl_params * param,struct mlx5_core_dev * mdev)755 mlx5e_rl_set_default_params(struct mlx5e_rl_params *param,
756 struct mlx5_core_dev *mdev)
757 {
758 /* ratelimit workers */
759 param->tx_worker_threads_def = mdev->priv.eq_table.num_comp_vectors;
760 param->tx_worker_threads_max = MLX5E_RL_MAX_WORKERS;
761
762 /* range check */
763 if (param->tx_worker_threads_def == 0 ||
764 param->tx_worker_threads_def > param->tx_worker_threads_max)
765 param->tx_worker_threads_def = param->tx_worker_threads_max;
766
767 /* ratelimit channels */
768 param->tx_channels_per_worker_def = MLX5E_RL_MAX_SQS /
769 param->tx_worker_threads_def;
770 param->tx_channels_per_worker_max = MLX5E_RL_MAX_SQS;
771
772 /* range check */
773 if (param->tx_channels_per_worker_def > MLX5E_RL_DEF_SQ_PER_WORKER)
774 param->tx_channels_per_worker_def = MLX5E_RL_DEF_SQ_PER_WORKER;
775
776 /* set default burst size */
777 param->tx_burst_size = 4; /* MTUs */
778
779 /*
780 * Set maximum burst size
781 *
782 * The burst size is multiplied by the MTU and clamped to the
783 * range 0 ... 65535 bytes inclusivly before fed into the
784 * firmware.
785 *
786 * NOTE: If the burst size or MTU is changed only ratelimit
787 * connections made after the change will use the new burst
788 * size.
789 */
790 param->tx_burst_size_max = 255;
791
792 /* get firmware rate limits in 1000bit/s and convert them to bit/s */
793 param->tx_limit_min = mdev->priv.rl_table.min_rate * 1000ULL;
794 param->tx_limit_max = mdev->priv.rl_table.max_rate * 1000ULL;
795
796 /* ratelimit table size */
797 param->tx_rates_max = mdev->priv.rl_table.max_size;
798
799 /* range check */
800 if (param->tx_rates_max > MLX5E_RL_MAX_TX_RATES)
801 param->tx_rates_max = MLX5E_RL_MAX_TX_RATES;
802
803 /* set default number of rates */
804 param->tx_rates_def = param->tx_rates_max;
805
806 /* set maximum allowed rate deviation */
807 if (param->tx_limit_max != 0) {
808 /*
809 * Make sure the deviation multiplication doesn't
810 * overflow unsigned 64-bit:
811 */
812 param->tx_allowed_deviation_max = -1ULL /
813 param->tx_limit_max;
814 }
815 /* set default rate deviation */
816 param->tx_allowed_deviation = 50; /* 5.0% */
817
818 /* channel parameters */
819 param->tx_queue_size = (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE);
820 param->tx_coalesce_usecs = MLX5E_RL_TX_COAL_USEC_DEFAULT;
821 param->tx_coalesce_pkts = MLX5E_RL_TX_COAL_PKTS_DEFAULT;
822 param->tx_coalesce_mode = MLX5E_RL_TX_COAL_MODE_DEFAULT;
823 param->tx_completion_fact = MLX5E_RL_TX_COMP_FACT_DEFAULT;
824 }
825
826 static const char *mlx5e_rl_params_desc[] = {
827 MLX5E_RL_PARAMS(MLX5E_STATS_DESC)
828 };
829
830 static const char *mlx5e_rl_table_params_desc[] = {
831 MLX5E_RL_TABLE_PARAMS(MLX5E_STATS_DESC)
832 };
833
834 static const char *mlx5e_rl_stats_desc[] = {
835 MLX5E_RL_STATS(MLX5E_STATS_DESC)
836 };
837
838 int
mlx5e_rl_init(struct mlx5e_priv * priv)839 mlx5e_rl_init(struct mlx5e_priv *priv)
840 {
841 struct mlx5e_rl_priv_data *rl = &priv->rl;
842 struct sysctl_oid *node;
843 struct sysctl_oid *stats;
844 char buf[64];
845 uint64_t i;
846 uint64_t j;
847 int error;
848
849 /* check if there is support for packet pacing */
850 if (!MLX5_CAP_GEN(priv->mdev, qos) || !MLX5_CAP_QOS(priv->mdev, packet_pacing))
851 return (0);
852
853 rl->priv = priv;
854
855 sysctl_ctx_init(&rl->ctx);
856
857 sx_init(&rl->rl_sxlock, "ratelimit-sxlock");
858
859 /* open own TIS domain for ratelimit SQs */
860 error = mlx5e_rl_open_tis(priv);
861 if (error)
862 goto done;
863
864 /* setup default value for parameters */
865 mlx5e_rl_set_default_params(&rl->param, priv->mdev);
866
867 /* update the completion factor */
868 mlx5e_rl_sync_tx_completion_fact(rl);
869
870 /* create root node */
871 node = SYSCTL_ADD_NODE(&rl->ctx,
872 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
873 "rate_limit", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Rate limiting support");
874
875 if (node != NULL) {
876 /* create SYSCTLs */
877 for (i = 0; i != MLX5E_RL_PARAMS_NUM; i++) {
878 mlx5e_rl_sysctl_add_u64_oid(rl,
879 MLX5E_RL_PARAMS_INDEX(arg[i]),
880 node, mlx5e_rl_params_desc[2 * i],
881 mlx5e_rl_params_desc[2 * i + 1]);
882 }
883
884 stats = SYSCTL_ADD_NODE(&rl->ctx, SYSCTL_CHILDREN(node),
885 OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
886 "Rate limiting statistics");
887 if (stats != NULL) {
888 /* create SYSCTLs */
889 for (i = 0; i != MLX5E_RL_STATS_NUM; i++) {
890 mlx5e_rl_sysctl_add_stats_u64_oid(rl, i,
891 stats, mlx5e_rl_stats_desc[2 * i],
892 mlx5e_rl_stats_desc[2 * i + 1]);
893 }
894 }
895 }
896
897 /* allocate workers array */
898 rl->workers = malloc(sizeof(rl->workers[0]) *
899 rl->param.tx_worker_threads_def, M_MLX5EN, M_WAITOK | M_ZERO);
900
901 /* allocate rate limit array */
902 rl->rate_limit_table = malloc(sizeof(rl->rate_limit_table[0]) *
903 rl->param.tx_rates_def, M_MLX5EN, M_WAITOK | M_ZERO);
904
905 if (node != NULL) {
906 /* create more SYSCTls */
907 SYSCTL_ADD_PROC(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO,
908 "tx_rate_show", CTLTYPE_STRING | CTLFLAG_RD |
909 CTLFLAG_MPSAFE, rl, 0, &mlx5e_rl_sysctl_show_rate_table,
910 "A", "Show table of all configured TX rates");
911
912 /* try to fetch rate table from kernel environment */
913 for (i = 0; i != rl->param.tx_rates_def; i++) {
914 /* compute path for tunable */
915 snprintf(buf, sizeof(buf), "dev.mce.%d.rate_limit.tx_rate_add_%d",
916 device_get_unit(priv->mdev->pdev->dev.bsddev), (int)i);
917 if (TUNABLE_QUAD_FETCH(buf, &j))
918 mlx5e_rl_tx_limit_add(rl, j);
919 }
920
921 /* setup rate table sysctls */
922 for (i = 0; i != MLX5E_RL_TABLE_PARAMS_NUM; i++) {
923 mlx5e_rl_sysctl_add_u64_oid(rl,
924 MLX5E_RL_PARAMS_INDEX(table_arg[i]),
925 node, mlx5e_rl_table_params_desc[2 * i],
926 mlx5e_rl_table_params_desc[2 * i + 1]);
927 }
928 }
929
930 for (j = 0; j < rl->param.tx_worker_threads_def; j++) {
931 struct mlx5e_rl_worker *rlw = rl->workers + j;
932
933 rlw->priv = priv;
934
935 cv_init(&rlw->cv, "mlx5-worker-cv");
936 mtx_init(&rlw->mtx, "mlx5-worker-mtx", NULL, MTX_DEF);
937 STAILQ_INIT(&rlw->index_list_head);
938 STAILQ_INIT(&rlw->process_head);
939
940 rlw->channels = malloc(sizeof(rlw->channels[0]) *
941 rl->param.tx_channels_per_worker_def, M_MLX5EN, M_WAITOK | M_ZERO);
942
943 MLX5E_RL_WORKER_LOCK(rlw);
944 for (i = 0; i < rl->param.tx_channels_per_worker_def; i++) {
945 struct mlx5e_rl_channel *channel = rlw->channels + i;
946 channel->worker = rlw;
947 channel->tag.type = IF_SND_TAG_TYPE_RATE_LIMIT;
948 STAILQ_INSERT_TAIL(&rlw->index_list_head, channel, entry);
949 }
950 MLX5E_RL_WORKER_UNLOCK(rlw);
951 }
952
953 PRIV_LOCK(priv);
954 error = mlx5e_rl_open_workers(priv);
955 PRIV_UNLOCK(priv);
956
957 if (error != 0) {
958 mlx5_en_err(priv->ifp,
959 "mlx5e_rl_open_workers failed: %d\n", error);
960 }
961
962 return (0);
963
964 done:
965 sysctl_ctx_free(&rl->ctx);
966 sx_destroy(&rl->rl_sxlock);
967 return (error);
968 }
969
970 static int
mlx5e_rl_open_workers(struct mlx5e_priv * priv)971 mlx5e_rl_open_workers(struct mlx5e_priv *priv)
972 {
973 struct mlx5e_rl_priv_data *rl = &priv->rl;
974 struct thread *rl_thread = NULL;
975 struct proc *rl_proc = NULL;
976 uint64_t j;
977 int error;
978
979 if (priv->gone || rl->opened)
980 return (-EINVAL);
981
982 MLX5E_RL_WLOCK(rl);
983 /* compute channel parameters once */
984 mlx5e_rl_build_channel_param(rl, &rl->chan_param);
985 MLX5E_RL_WUNLOCK(rl);
986
987 for (j = 0; j < rl->param.tx_worker_threads_def; j++) {
988 struct mlx5e_rl_worker *rlw = rl->workers + j;
989
990 /* start worker thread */
991 error = kproc_kthread_add(mlx5e_rl_worker, rlw, &rl_proc, &rl_thread,
992 RFHIGHPID, 0, "mlx5-ratelimit", "mlx5-rl-worker-thread-%d", (int)j);
993 if (error != 0) {
994 mlx5_en_err(rl->priv->ifp,
995 "kproc_kthread_add failed: %d\n", error);
996 rlw->worker_done = 1;
997 }
998 }
999
1000 rl->opened = 1;
1001
1002 return (0);
1003 }
1004
1005 static void
mlx5e_rl_close_workers(struct mlx5e_priv * priv)1006 mlx5e_rl_close_workers(struct mlx5e_priv *priv)
1007 {
1008 struct mlx5e_rl_priv_data *rl = &priv->rl;
1009 uint64_t y;
1010
1011 if (rl->opened == 0)
1012 return;
1013
1014 /* tear down worker threads simultaneously */
1015 for (y = 0; y < rl->param.tx_worker_threads_def; y++) {
1016 struct mlx5e_rl_worker *rlw = rl->workers + y;
1017
1018 /* tear down worker before freeing SQs */
1019 MLX5E_RL_WORKER_LOCK(rlw);
1020 if (rlw->worker_done == 0) {
1021 rlw->worker_done = 1;
1022 cv_broadcast(&rlw->cv);
1023 } else {
1024 /* XXX thread not started */
1025 rlw->worker_done = 0;
1026 }
1027 MLX5E_RL_WORKER_UNLOCK(rlw);
1028 }
1029
1030 /* wait for worker threads to exit */
1031 for (y = 0; y < rl->param.tx_worker_threads_def; y++) {
1032 struct mlx5e_rl_worker *rlw = rl->workers + y;
1033
1034 /* tear down worker before freeing SQs */
1035 MLX5E_RL_WORKER_LOCK(rlw);
1036 while (rlw->worker_done != 0)
1037 cv_wait(&rlw->cv, &rlw->mtx);
1038 MLX5E_RL_WORKER_UNLOCK(rlw);
1039 }
1040
1041 rl->opened = 0;
1042 }
1043
1044 static void
mlx5e_rl_reset_rates(struct mlx5e_rl_priv_data * rl)1045 mlx5e_rl_reset_rates(struct mlx5e_rl_priv_data *rl)
1046 {
1047 unsigned x;
1048
1049 MLX5E_RL_WLOCK(rl);
1050 for (x = 0; x != rl->param.tx_rates_def; x++)
1051 rl->rate_limit_table[x] = 0;
1052 MLX5E_RL_WUNLOCK(rl);
1053 }
1054
1055 void
mlx5e_rl_cleanup(struct mlx5e_priv * priv)1056 mlx5e_rl_cleanup(struct mlx5e_priv *priv)
1057 {
1058 struct mlx5e_rl_priv_data *rl = &priv->rl;
1059 uint64_t y;
1060
1061 /* check if there is support for packet pacing */
1062 if (!MLX5_CAP_GEN(priv->mdev, qos) || !MLX5_CAP_QOS(priv->mdev, packet_pacing))
1063 return;
1064
1065 /* TODO check if there is support for packet pacing */
1066
1067 sysctl_ctx_free(&rl->ctx);
1068
1069 PRIV_LOCK(priv);
1070 mlx5e_rl_close_workers(priv);
1071 PRIV_UNLOCK(priv);
1072
1073 mlx5e_rl_reset_rates(rl);
1074
1075 /* close TIS domain */
1076 mlx5e_rl_close_tis(priv);
1077
1078 for (y = 0; y < rl->param.tx_worker_threads_def; y++) {
1079 struct mlx5e_rl_worker *rlw = rl->workers + y;
1080
1081 cv_destroy(&rlw->cv);
1082 mtx_destroy(&rlw->mtx);
1083 free(rlw->channels, M_MLX5EN);
1084 }
1085 free(rl->rate_limit_table, M_MLX5EN);
1086 free(rl->workers, M_MLX5EN);
1087 sx_destroy(&rl->rl_sxlock);
1088 }
1089
1090 static void
mlx5e_rlw_queue_channel_locked(struct mlx5e_rl_worker * rlw,struct mlx5e_rl_channel * channel)1091 mlx5e_rlw_queue_channel_locked(struct mlx5e_rl_worker *rlw,
1092 struct mlx5e_rl_channel *channel)
1093 {
1094 STAILQ_INSERT_TAIL(&rlw->process_head, channel, entry);
1095 cv_broadcast(&rlw->cv);
1096 }
1097
1098 static void
mlx5e_rl_free(struct mlx5e_rl_worker * rlw,struct mlx5e_rl_channel * channel)1099 mlx5e_rl_free(struct mlx5e_rl_worker *rlw, struct mlx5e_rl_channel *channel)
1100 {
1101 if (channel == NULL)
1102 return;
1103
1104 MLX5E_RL_WORKER_LOCK(rlw);
1105 switch (channel->state) {
1106 case MLX5E_RL_ST_MODIFY:
1107 channel->state = MLX5E_RL_ST_DESTROY;
1108 break;
1109 case MLX5E_RL_ST_USED:
1110 channel->state = MLX5E_RL_ST_DESTROY;
1111 mlx5e_rlw_queue_channel_locked(rlw, channel);
1112 break;
1113 default:
1114 break;
1115 }
1116 MLX5E_RL_WORKER_UNLOCK(rlw);
1117 }
1118
1119 static int
mlx5e_rl_modify(struct mlx5e_rl_worker * rlw,struct mlx5e_rl_channel * channel,uint64_t rate)1120 mlx5e_rl_modify(struct mlx5e_rl_worker *rlw, struct mlx5e_rl_channel *channel, uint64_t rate)
1121 {
1122
1123 MLX5E_RL_WORKER_LOCK(rlw);
1124 channel->new_rate = rate;
1125 switch (channel->state) {
1126 case MLX5E_RL_ST_USED:
1127 channel->state = MLX5E_RL_ST_MODIFY;
1128 mlx5e_rlw_queue_channel_locked(rlw, channel);
1129 break;
1130 default:
1131 break;
1132 }
1133 MLX5E_RL_WORKER_UNLOCK(rlw);
1134
1135 return (0);
1136 }
1137
1138 static int
mlx5e_rl_query(struct mlx5e_rl_worker * rlw,struct mlx5e_rl_channel * channel,union if_snd_tag_query_params * params)1139 mlx5e_rl_query(struct mlx5e_rl_worker *rlw, struct mlx5e_rl_channel *channel,
1140 union if_snd_tag_query_params *params)
1141 {
1142 int retval;
1143
1144 MLX5E_RL_WORKER_LOCK(rlw);
1145 switch (channel->state) {
1146 case MLX5E_RL_ST_USED:
1147 params->rate_limit.max_rate = channel->last_rate;
1148 params->rate_limit.queue_level = mlx5e_sq_queue_level(channel->sq);
1149 retval = 0;
1150 break;
1151 case MLX5E_RL_ST_MODIFY:
1152 params->rate_limit.max_rate = channel->last_rate;
1153 params->rate_limit.queue_level = mlx5e_sq_queue_level(channel->sq);
1154 retval = EBUSY;
1155 break;
1156 default:
1157 retval = EINVAL;
1158 break;
1159 }
1160 MLX5E_RL_WORKER_UNLOCK(rlw);
1161
1162 return (retval);
1163 }
1164
1165 static int
mlx5e_find_available_tx_ring_index(struct mlx5e_rl_worker * rlw,struct mlx5e_rl_channel ** pchannel)1166 mlx5e_find_available_tx_ring_index(struct mlx5e_rl_worker *rlw,
1167 struct mlx5e_rl_channel **pchannel)
1168 {
1169 struct mlx5e_rl_channel *channel;
1170 int retval = ENOMEM;
1171
1172 MLX5E_RL_WORKER_LOCK(rlw);
1173 /* Check for available channel in free list */
1174 if ((channel = STAILQ_FIRST(&rlw->index_list_head)) != NULL) {
1175 retval = 0;
1176 /* Remove head index from available list */
1177 STAILQ_REMOVE_HEAD(&rlw->index_list_head, entry);
1178 channel->state = MLX5E_RL_ST_USED;
1179 atomic_add_64(&rlw->priv->rl.stats.tx_active_connections, 1ULL);
1180 } else {
1181 atomic_add_64(&rlw->priv->rl.stats.tx_available_resource_failure, 1ULL);
1182 }
1183 MLX5E_RL_WORKER_UNLOCK(rlw);
1184
1185 *pchannel = channel;
1186 #ifdef RATELIMIT_DEBUG
1187 mlx5_en_info(rlw->priv->ifp,
1188 "Channel pointer for rate limit connection is %p\n", channel);
1189 #endif
1190 return (retval);
1191 }
1192
1193 int
mlx5e_rl_snd_tag_alloc(struct ifnet * ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** ppmt)1194 mlx5e_rl_snd_tag_alloc(struct ifnet *ifp,
1195 union if_snd_tag_alloc_params *params,
1196 struct m_snd_tag **ppmt)
1197 {
1198 struct mlx5e_rl_channel *channel;
1199 struct mlx5e_rl_worker *rlw;
1200 struct mlx5e_priv *priv;
1201 int error;
1202
1203 priv = ifp->if_softc;
1204
1205 /* check if there is support for packet pacing or if device is going away */
1206 if (!MLX5_CAP_GEN(priv->mdev, qos) ||
1207 !MLX5_CAP_QOS(priv->mdev, packet_pacing) || priv->gone ||
1208 params->rate_limit.hdr.type != IF_SND_TAG_TYPE_RATE_LIMIT)
1209 return (EOPNOTSUPP);
1210
1211 /* compute worker thread this TCP connection belongs to */
1212 rlw = priv->rl.workers + ((params->rate_limit.hdr.flowid % 128) %
1213 priv->rl.param.tx_worker_threads_def);
1214
1215 error = mlx5e_find_available_tx_ring_index(rlw, &channel);
1216 if (error != 0)
1217 goto done;
1218
1219 error = mlx5e_rl_modify(rlw, channel, params->rate_limit.max_rate);
1220 if (error != 0) {
1221 mlx5e_rl_free(rlw, channel);
1222 goto done;
1223 }
1224
1225 /* store pointer to mbuf tag */
1226 MPASS(channel->tag.refcount == 0);
1227 m_snd_tag_init(&channel->tag, ifp, IF_SND_TAG_TYPE_RATE_LIMIT);
1228 *ppmt = &channel->tag;
1229 done:
1230 return (error);
1231 }
1232
1233
1234 int
mlx5e_rl_snd_tag_modify(struct m_snd_tag * pmt,union if_snd_tag_modify_params * params)1235 mlx5e_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
1236 {
1237 struct mlx5e_rl_channel *channel =
1238 container_of(pmt, struct mlx5e_rl_channel, tag);
1239
1240 return (mlx5e_rl_modify(channel->worker, channel, params->rate_limit.max_rate));
1241 }
1242
1243 int
mlx5e_rl_snd_tag_query(struct m_snd_tag * pmt,union if_snd_tag_query_params * params)1244 mlx5e_rl_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
1245 {
1246 struct mlx5e_rl_channel *channel =
1247 container_of(pmt, struct mlx5e_rl_channel, tag);
1248
1249 return (mlx5e_rl_query(channel->worker, channel, params));
1250 }
1251
1252 void
mlx5e_rl_snd_tag_free(struct m_snd_tag * pmt)1253 mlx5e_rl_snd_tag_free(struct m_snd_tag *pmt)
1254 {
1255 struct mlx5e_rl_channel *channel =
1256 container_of(pmt, struct mlx5e_rl_channel, tag);
1257
1258 mlx5e_rl_free(channel->worker, channel);
1259 }
1260
1261 static int
mlx5e_rl_sysctl_show_rate_table(SYSCTL_HANDLER_ARGS)1262 mlx5e_rl_sysctl_show_rate_table(SYSCTL_HANDLER_ARGS)
1263 {
1264 struct mlx5e_rl_priv_data *rl = arg1;
1265 struct mlx5e_priv *priv = rl->priv;
1266 struct sbuf sbuf;
1267 unsigned x;
1268 int error;
1269
1270 error = sysctl_wire_old_buffer(req, 0);
1271 if (error != 0)
1272 return (error);
1273
1274 PRIV_LOCK(priv);
1275
1276 sbuf_new_for_sysctl(&sbuf, NULL, 128 * rl->param.tx_rates_def, req);
1277
1278 sbuf_printf(&sbuf,
1279 "\n\n" "\t" "ENTRY" "\t" "BURST" "\t" "RATE [bit/s]\n"
1280 "\t" "--------------------------------------------\n");
1281
1282 MLX5E_RL_RLOCK(rl);
1283 for (x = 0; x != rl->param.tx_rates_def; x++) {
1284 if (rl->rate_limit_table[x] == 0)
1285 continue;
1286
1287 sbuf_printf(&sbuf, "\t" "%3u" "\t" "%3u" "\t" "%lld\n",
1288 x, (unsigned)rl->param.tx_burst_size,
1289 (long long)rl->rate_limit_table[x]);
1290 }
1291 MLX5E_RL_RUNLOCK(rl);
1292
1293 error = sbuf_finish(&sbuf);
1294 sbuf_delete(&sbuf);
1295
1296 PRIV_UNLOCK(priv);
1297
1298 return (error);
1299 }
1300
1301 static int
mlx5e_rl_refresh_channel_params(struct mlx5e_rl_priv_data * rl)1302 mlx5e_rl_refresh_channel_params(struct mlx5e_rl_priv_data *rl)
1303 {
1304 uint64_t x;
1305 uint64_t y;
1306
1307 MLX5E_RL_WLOCK(rl);
1308 /* compute channel parameters once */
1309 mlx5e_rl_build_channel_param(rl, &rl->chan_param);
1310 MLX5E_RL_WUNLOCK(rl);
1311
1312 for (y = 0; y != rl->param.tx_worker_threads_def; y++) {
1313 struct mlx5e_rl_worker *rlw = rl->workers + y;
1314
1315 for (x = 0; x != rl->param.tx_channels_per_worker_def; x++) {
1316 struct mlx5e_rl_channel *channel;
1317 struct mlx5e_sq *sq;
1318
1319 channel = rlw->channels + x;
1320 sq = channel->sq;
1321
1322 if (sq == NULL)
1323 continue;
1324
1325 if (MLX5_CAP_GEN(rl->priv->mdev, cq_period_mode_modify)) {
1326 mlx5_core_modify_cq_moderation_mode(rl->priv->mdev, &sq->cq.mcq,
1327 rl->param.tx_coalesce_usecs,
1328 rl->param.tx_coalesce_pkts,
1329 rl->param.tx_coalesce_mode);
1330 } else {
1331 mlx5_core_modify_cq_moderation(rl->priv->mdev, &sq->cq.mcq,
1332 rl->param.tx_coalesce_usecs,
1333 rl->param.tx_coalesce_pkts);
1334 }
1335 }
1336 }
1337 return (0);
1338 }
1339
1340 void
mlx5e_rl_refresh_sq_inline(struct mlx5e_rl_priv_data * rl)1341 mlx5e_rl_refresh_sq_inline(struct mlx5e_rl_priv_data *rl)
1342 {
1343 uint64_t x;
1344 uint64_t y;
1345
1346 for (y = 0; y != rl->param.tx_worker_threads_def; y++) {
1347 struct mlx5e_rl_worker *rlw = rl->workers + y;
1348
1349 for (x = 0; x != rl->param.tx_channels_per_worker_def; x++) {
1350 struct mlx5e_rl_channel *channel;
1351 struct mlx5e_sq *sq;
1352
1353 channel = rlw->channels + x;
1354 sq = channel->sq;
1355
1356 if (sq == NULL)
1357 continue;
1358
1359 mtx_lock(&sq->lock);
1360 mlx5e_update_sq_inline(sq);
1361 mtx_unlock(&sq->lock);
1362 }
1363 }
1364 }
1365
1366 static int
mlx5e_rl_tx_limit_add(struct mlx5e_rl_priv_data * rl,uint64_t value)1367 mlx5e_rl_tx_limit_add(struct mlx5e_rl_priv_data *rl, uint64_t value)
1368 {
1369 unsigned x;
1370 int error;
1371
1372 if (value < 1000 ||
1373 mlx5_rl_is_in_range(rl->priv->mdev, howmany(value, 1000), 0) == 0)
1374 return (EINVAL);
1375
1376 MLX5E_RL_WLOCK(rl);
1377 error = ENOMEM;
1378
1379 /* check if rate already exists */
1380 for (x = 0; x != rl->param.tx_rates_def; x++) {
1381 if (rl->rate_limit_table[x] != value)
1382 continue;
1383 error = EEXIST;
1384 break;
1385 }
1386
1387 /* check if there is a free rate entry */
1388 if (x == rl->param.tx_rates_def) {
1389 for (x = 0; x != rl->param.tx_rates_def; x++) {
1390 if (rl->rate_limit_table[x] != 0)
1391 continue;
1392 rl->rate_limit_table[x] = value;
1393 error = 0;
1394 break;
1395 }
1396 }
1397 MLX5E_RL_WUNLOCK(rl);
1398
1399 return (error);
1400 }
1401
1402 static int
mlx5e_rl_tx_limit_clr(struct mlx5e_rl_priv_data * rl,uint64_t value)1403 mlx5e_rl_tx_limit_clr(struct mlx5e_rl_priv_data *rl, uint64_t value)
1404 {
1405 unsigned x;
1406 int error;
1407
1408 if (value == 0)
1409 return (EINVAL);
1410
1411 MLX5E_RL_WLOCK(rl);
1412
1413 /* check if rate already exists */
1414 for (x = 0; x != rl->param.tx_rates_def; x++) {
1415 if (rl->rate_limit_table[x] != value)
1416 continue;
1417 /* free up rate */
1418 rl->rate_limit_table[x] = 0;
1419 break;
1420 }
1421
1422 /* check if there is a free rate entry */
1423 if (x == rl->param.tx_rates_def)
1424 error = ENOENT;
1425 else
1426 error = 0;
1427 MLX5E_RL_WUNLOCK(rl);
1428
1429 return (error);
1430 }
1431
1432 static int
mlx5e_rl_sysctl_handler(SYSCTL_HANDLER_ARGS)1433 mlx5e_rl_sysctl_handler(SYSCTL_HANDLER_ARGS)
1434 {
1435 struct mlx5e_rl_priv_data *rl = arg1;
1436 struct mlx5e_priv *priv = rl->priv;
1437 unsigned mode_modify;
1438 unsigned was_opened;
1439 uint64_t value;
1440 uint64_t old;
1441 int error;
1442
1443 PRIV_LOCK(priv);
1444
1445 MLX5E_RL_RLOCK(rl);
1446 value = rl->param.arg[arg2];
1447 MLX5E_RL_RUNLOCK(rl);
1448
1449 if (req != NULL) {
1450 old = value;
1451 error = sysctl_handle_64(oidp, &value, 0, req);
1452 if (error || req->newptr == NULL ||
1453 value == rl->param.arg[arg2])
1454 goto done;
1455 } else {
1456 old = 0;
1457 error = 0;
1458 }
1459
1460 /* check if device is gone */
1461 if (priv->gone) {
1462 error = ENXIO;
1463 goto done;
1464 }
1465 was_opened = rl->opened;
1466 mode_modify = MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify);
1467
1468 switch (MLX5E_RL_PARAMS_INDEX(arg[arg2])) {
1469 case MLX5E_RL_PARAMS_INDEX(tx_worker_threads_def):
1470 if (value > rl->param.tx_worker_threads_max)
1471 value = rl->param.tx_worker_threads_max;
1472 else if (value < 1)
1473 value = 1;
1474
1475 /* store new value */
1476 rl->param.arg[arg2] = value;
1477 break;
1478
1479 case MLX5E_RL_PARAMS_INDEX(tx_channels_per_worker_def):
1480 if (value > rl->param.tx_channels_per_worker_max)
1481 value = rl->param.tx_channels_per_worker_max;
1482 else if (value < 1)
1483 value = 1;
1484
1485 /* store new value */
1486 rl->param.arg[arg2] = value;
1487 break;
1488
1489 case MLX5E_RL_PARAMS_INDEX(tx_rates_def):
1490 if (value > rl->param.tx_rates_max)
1491 value = rl->param.tx_rates_max;
1492 else if (value < 1)
1493 value = 1;
1494
1495 /* store new value */
1496 rl->param.arg[arg2] = value;
1497 break;
1498
1499 case MLX5E_RL_PARAMS_INDEX(tx_coalesce_usecs):
1500 /* range check */
1501 if (value < 1)
1502 value = 0;
1503 else if (value > MLX5E_FLD_MAX(cqc, cq_period))
1504 value = MLX5E_FLD_MAX(cqc, cq_period);
1505
1506 /* store new value */
1507 rl->param.arg[arg2] = value;
1508
1509 /* check to avoid down and up the network interface */
1510 if (was_opened)
1511 error = mlx5e_rl_refresh_channel_params(rl);
1512 break;
1513
1514 case MLX5E_RL_PARAMS_INDEX(tx_coalesce_pkts):
1515 /* import TX coal pkts */
1516 if (value < 1)
1517 value = 0;
1518 else if (value > MLX5E_FLD_MAX(cqc, cq_max_count))
1519 value = MLX5E_FLD_MAX(cqc, cq_max_count);
1520
1521 /* store new value */
1522 rl->param.arg[arg2] = value;
1523
1524 /* check to avoid down and up the network interface */
1525 if (was_opened)
1526 error = mlx5e_rl_refresh_channel_params(rl);
1527 break;
1528
1529 case MLX5E_RL_PARAMS_INDEX(tx_coalesce_mode):
1530 /* network interface must be down */
1531 if (was_opened != 0 && mode_modify == 0)
1532 mlx5e_rl_close_workers(priv);
1533
1534 /* import TX coalesce mode */
1535 if (value != 0)
1536 value = 1;
1537
1538 /* store new value */
1539 rl->param.arg[arg2] = value;
1540
1541 /* restart network interface, if any */
1542 if (was_opened != 0) {
1543 if (mode_modify == 0)
1544 mlx5e_rl_open_workers(priv);
1545 else
1546 error = mlx5e_rl_refresh_channel_params(rl);
1547 }
1548 break;
1549
1550 case MLX5E_RL_PARAMS_INDEX(tx_queue_size):
1551 /* network interface must be down */
1552 if (was_opened)
1553 mlx5e_rl_close_workers(priv);
1554
1555 /* import TX queue size */
1556 if (value < (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE))
1557 value = (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE);
1558 else if (value > priv->params_ethtool.tx_queue_size_max)
1559 value = priv->params_ethtool.tx_queue_size_max;
1560
1561 /* store actual TX queue size */
1562 value = 1ULL << order_base_2(value);
1563
1564 /* store new value */
1565 rl->param.arg[arg2] = value;
1566
1567 /* verify TX completion factor */
1568 mlx5e_rl_sync_tx_completion_fact(rl);
1569
1570 /* restart network interface, if any */
1571 if (was_opened)
1572 mlx5e_rl_open_workers(priv);
1573 break;
1574
1575 case MLX5E_RL_PARAMS_INDEX(tx_completion_fact):
1576 /* network interface must be down */
1577 if (was_opened)
1578 mlx5e_rl_close_workers(priv);
1579
1580 /* store new value */
1581 rl->param.arg[arg2] = value;
1582
1583 /* verify parameter */
1584 mlx5e_rl_sync_tx_completion_fact(rl);
1585
1586 /* restart network interface, if any */
1587 if (was_opened)
1588 mlx5e_rl_open_workers(priv);
1589 break;
1590
1591 case MLX5E_RL_PARAMS_INDEX(tx_limit_add):
1592 error = mlx5e_rl_tx_limit_add(rl, value);
1593 break;
1594
1595 case MLX5E_RL_PARAMS_INDEX(tx_limit_clr):
1596 error = mlx5e_rl_tx_limit_clr(rl, value);
1597 break;
1598
1599 case MLX5E_RL_PARAMS_INDEX(tx_allowed_deviation):
1600 /* range check */
1601 if (value > rl->param.tx_allowed_deviation_max)
1602 value = rl->param.tx_allowed_deviation_max;
1603 else if (value < rl->param.tx_allowed_deviation_min)
1604 value = rl->param.tx_allowed_deviation_min;
1605
1606 MLX5E_RL_WLOCK(rl);
1607 rl->param.arg[arg2] = value;
1608 MLX5E_RL_WUNLOCK(rl);
1609 break;
1610
1611 case MLX5E_RL_PARAMS_INDEX(tx_burst_size):
1612 /* range check */
1613 if (value > rl->param.tx_burst_size_max)
1614 value = rl->param.tx_burst_size_max;
1615 else if (value < rl->param.tx_burst_size_min)
1616 value = rl->param.tx_burst_size_min;
1617
1618 MLX5E_RL_WLOCK(rl);
1619 rl->param.arg[arg2] = value;
1620 MLX5E_RL_WUNLOCK(rl);
1621 break;
1622
1623 default:
1624 break;
1625 }
1626 done:
1627 PRIV_UNLOCK(priv);
1628 return (error);
1629 }
1630
1631 static void
mlx5e_rl_sysctl_add_u64_oid(struct mlx5e_rl_priv_data * rl,unsigned x,struct sysctl_oid * node,const char * name,const char * desc)1632 mlx5e_rl_sysctl_add_u64_oid(struct mlx5e_rl_priv_data *rl, unsigned x,
1633 struct sysctl_oid *node, const char *name, const char *desc)
1634 {
1635 /*
1636 * NOTE: In FreeBSD-11 and newer the CTLFLAG_RWTUN flag will
1637 * take care of loading default sysctl value from the kernel
1638 * environment, if any:
1639 */
1640 if (strstr(name, "_max") != 0 || strstr(name, "_min") != 0) {
1641 /* read-only SYSCTLs */
1642 SYSCTL_ADD_PROC(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO,
1643 name, CTLTYPE_U64 | CTLFLAG_RD |
1644 CTLFLAG_MPSAFE, rl, x, &mlx5e_rl_sysctl_handler, "QU", desc);
1645 } else {
1646 if (strstr(name, "_def") != 0) {
1647 #ifdef RATELIMIT_DEBUG
1648 /* tunable read-only advanced SYSCTLs */
1649 SYSCTL_ADD_PROC(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO,
1650 name, CTLTYPE_U64 | CTLFLAG_RDTUN |
1651 CTLFLAG_MPSAFE, rl, x, &mlx5e_rl_sysctl_handler, "QU", desc);
1652 #endif
1653 } else {
1654 /* read-write SYSCTLs */
1655 SYSCTL_ADD_PROC(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO,
1656 name, CTLTYPE_U64 | CTLFLAG_RWTUN |
1657 CTLFLAG_MPSAFE, rl, x, &mlx5e_rl_sysctl_handler, "QU", desc);
1658 }
1659 }
1660 }
1661
1662 static void
mlx5e_rl_sysctl_add_stats_u64_oid(struct mlx5e_rl_priv_data * rl,unsigned x,struct sysctl_oid * node,const char * name,const char * desc)1663 mlx5e_rl_sysctl_add_stats_u64_oid(struct mlx5e_rl_priv_data *rl, unsigned x,
1664 struct sysctl_oid *node, const char *name, const char *desc)
1665 {
1666 /* read-only SYSCTLs */
1667 SYSCTL_ADD_U64(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO, name,
1668 CTLFLAG_RD, &rl->stats.arg[x], 0, desc);
1669 }
1670
1671 #else
1672
1673 int
mlx5e_rl_init(struct mlx5e_priv * priv)1674 mlx5e_rl_init(struct mlx5e_priv *priv)
1675 {
1676
1677 return (0);
1678 }
1679
1680 void
mlx5e_rl_cleanup(struct mlx5e_priv * priv)1681 mlx5e_rl_cleanup(struct mlx5e_priv *priv)
1682 {
1683 /* NOP */
1684 }
1685
1686 #endif /* RATELIMIT */
1687