1 /*-
2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD$
26 */
27
28 #include <dev/mlx5/driver.h>
29 #include <dev/mlx5/flow_table.h>
30 #include "mlx5_core.h"
31
32 struct mlx5_ftg {
33 struct mlx5_flow_table_group g;
34 u32 id;
35 u32 start_ix;
36 };
37
38 struct mlx5_flow_table {
39 struct mlx5_core_dev *dev;
40 u8 level;
41 u8 type;
42 u32 id;
43 u16 vport;
44 struct mutex mutex; /* sync bitmap alloc */
45 u16 num_groups;
46 struct mlx5_ftg *group;
47 unsigned long *bitmap;
48 u32 size;
49 };
50
mlx5_set_flow_entry_cmd(struct mlx5_flow_table * ft,u32 group_ix,u32 flow_index,void * flow_context)51 static int mlx5_set_flow_entry_cmd(struct mlx5_flow_table *ft, u32 group_ix,
52 u32 flow_index, void *flow_context)
53 {
54 u32 out[MLX5_ST_SZ_DW(set_fte_out)];
55 u32 *in;
56 void *in_flow_context;
57 int fcdls =
58 MLX5_GET(flow_context, flow_context, destination_list_size) *
59 MLX5_ST_SZ_BYTES(dest_format_struct);
60 int inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fcdls;
61 int err;
62
63 in = mlx5_vzalloc(inlen);
64 if (!in) {
65 mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
66 return -ENOMEM;
67 }
68
69 MLX5_SET(set_fte_in, in, vport_number, ft->vport);
70 MLX5_SET(set_fte_in, in, other_vport, !!ft->vport);
71 MLX5_SET(set_fte_in, in, table_type, ft->type);
72 MLX5_SET(set_fte_in, in, table_id, ft->id);
73 MLX5_SET(set_fte_in, in, flow_index, flow_index);
74 MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY);
75
76 in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context);
77 memcpy(in_flow_context, flow_context,
78 MLX5_ST_SZ_BYTES(flow_context) + fcdls);
79
80 MLX5_SET(flow_context, in_flow_context, group_id, ft->group[group_ix].id);
81
82 memset(out, 0, sizeof(out));
83 err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
84 sizeof(out));
85 kvfree(in);
86
87 return err;
88 }
89
mlx5_del_flow_entry_cmd(struct mlx5_flow_table * ft,u32 flow_index)90 static void mlx5_del_flow_entry_cmd(struct mlx5_flow_table *ft, u32 flow_index)
91 {
92 u32 in[MLX5_ST_SZ_DW(delete_fte_in)];
93 u32 out[MLX5_ST_SZ_DW(delete_fte_out)];
94
95 memset(in, 0, sizeof(in));
96 memset(out, 0, sizeof(out));
97
98 #define MLX5_SET_DFTEI(p, x, v) MLX5_SET(delete_fte_in, p, x, v)
99 MLX5_SET_DFTEI(in, vport_number, ft->vport);
100 MLX5_SET_DFTEI(in, other_vport, !!ft->vport);
101 MLX5_SET_DFTEI(in, table_type, ft->type);
102 MLX5_SET_DFTEI(in, table_id, ft->id);
103 MLX5_SET_DFTEI(in, flow_index, flow_index);
104 MLX5_SET_DFTEI(in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
105
106 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
107 }
108
mlx5_destroy_flow_group_cmd(struct mlx5_flow_table * ft,int i)109 static void mlx5_destroy_flow_group_cmd(struct mlx5_flow_table *ft, int i)
110 {
111 u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)];
112 u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)];
113
114 memset(in, 0, sizeof(in));
115 memset(out, 0, sizeof(out));
116
117 #define MLX5_SET_DFGI(p, x, v) MLX5_SET(destroy_flow_group_in, p, x, v)
118 MLX5_SET_DFGI(in, vport_number, ft->vport);
119 MLX5_SET_DFGI(in, other_vport, !!ft->vport);
120 MLX5_SET_DFGI(in, table_type, ft->type);
121 MLX5_SET_DFGI(in, table_id, ft->id);
122 MLX5_SET_DFGI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_GROUP);
123 MLX5_SET_DFGI(in, group_id, ft->group[i].id);
124 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
125 }
126
mlx5_create_flow_group_cmd(struct mlx5_flow_table * ft,int i)127 static int mlx5_create_flow_group_cmd(struct mlx5_flow_table *ft, int i)
128 {
129 u32 out[MLX5_ST_SZ_DW(create_flow_group_out)];
130 u32 *in;
131 void *in_match_criteria;
132 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
133 struct mlx5_flow_table_group *g = &ft->group[i].g;
134 u32 start_ix = ft->group[i].start_ix;
135 u32 end_ix = start_ix + (1 << g->log_sz) - 1;
136 int err;
137
138 in = mlx5_vzalloc(inlen);
139 if (!in) {
140 mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
141 return -ENOMEM;
142 }
143 in_match_criteria = MLX5_ADDR_OF(create_flow_group_in, in,
144 match_criteria);
145
146 memset(out, 0, sizeof(out));
147
148 #define MLX5_SET_CFGI(p, x, v) MLX5_SET(create_flow_group_in, p, x, v)
149 MLX5_SET_CFGI(in, vport_number, ft->vport);
150 MLX5_SET_CFGI(in, other_vport, !!ft->vport);
151 MLX5_SET_CFGI(in, table_type, ft->type);
152 MLX5_SET_CFGI(in, table_id, ft->id);
153 MLX5_SET_CFGI(in, opcode, MLX5_CMD_OP_CREATE_FLOW_GROUP);
154 MLX5_SET_CFGI(in, start_flow_index, start_ix);
155 MLX5_SET_CFGI(in, end_flow_index, end_ix);
156 MLX5_SET_CFGI(in, match_criteria_enable, g->match_criteria_enable);
157
158 memcpy(in_match_criteria, g->match_criteria,
159 MLX5_ST_SZ_BYTES(fte_match_param));
160
161 err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
162 sizeof(out));
163 if (!err)
164 ft->group[i].id = MLX5_GET(create_flow_group_out, out,
165 group_id);
166
167 kvfree(in);
168
169 return err;
170 }
171
mlx5_destroy_flow_table_groups(struct mlx5_flow_table * ft)172 static void mlx5_destroy_flow_table_groups(struct mlx5_flow_table *ft)
173 {
174 int i;
175
176 for (i = 0; i < ft->num_groups; i++)
177 mlx5_destroy_flow_group_cmd(ft, i);
178 }
179
mlx5_create_flow_table_groups(struct mlx5_flow_table * ft)180 static int mlx5_create_flow_table_groups(struct mlx5_flow_table *ft)
181 {
182 int err;
183 int i;
184
185 for (i = 0; i < ft->num_groups; i++) {
186 err = mlx5_create_flow_group_cmd(ft, i);
187 if (err)
188 goto err_destroy_flow_table_groups;
189 }
190
191 return 0;
192
193 err_destroy_flow_table_groups:
194 for (i--; i >= 0; i--)
195 mlx5_destroy_flow_group_cmd(ft, i);
196
197 return err;
198 }
199
mlx5_create_flow_table_cmd(struct mlx5_flow_table * ft)200 static int mlx5_create_flow_table_cmd(struct mlx5_flow_table *ft)
201 {
202 u32 in[MLX5_ST_SZ_DW(create_flow_table_in)];
203 u32 out[MLX5_ST_SZ_DW(create_flow_table_out)];
204 int err;
205
206 memset(in, 0, sizeof(in));
207
208 MLX5_SET(create_flow_table_in, in, vport_number, ft->vport);
209 MLX5_SET(create_flow_table_in, in, other_vport, !!ft->vport);
210 MLX5_SET(create_flow_table_in, in, table_type, ft->type);
211 MLX5_SET(create_flow_table_in, in, level, ft->level);
212 MLX5_SET(create_flow_table_in, in, log_size, order_base_2(ft->size));
213
214 MLX5_SET(create_flow_table_in, in, opcode,
215 MLX5_CMD_OP_CREATE_FLOW_TABLE);
216
217 memset(out, 0, sizeof(out));
218 err = mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out,
219 sizeof(out));
220 if (err)
221 return err;
222
223 ft->id = MLX5_GET(create_flow_table_out, out, table_id);
224
225 return 0;
226 }
227
mlx5_destroy_flow_table_cmd(struct mlx5_flow_table * ft)228 static void mlx5_destroy_flow_table_cmd(struct mlx5_flow_table *ft)
229 {
230 u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)];
231 u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)];
232
233 memset(in, 0, sizeof(in));
234 memset(out, 0, sizeof(out));
235
236 #define MLX5_SET_DFTI(p, x, v) MLX5_SET(destroy_flow_table_in, p, x, v)
237 MLX5_SET_DFTI(in, vport_number, ft->vport);
238 MLX5_SET_DFTI(in, other_vport, !!ft->vport);
239 MLX5_SET_DFTI(in, table_type, ft->type);
240 MLX5_SET_DFTI(in, table_id, ft->id);
241 MLX5_SET_DFTI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_TABLE);
242
243 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
244 }
245
mlx5_find_group(struct mlx5_flow_table * ft,u8 match_criteria_enable,u32 * match_criteria,int * group_ix)246 static int mlx5_find_group(struct mlx5_flow_table *ft, u8 match_criteria_enable,
247 u32 *match_criteria, int *group_ix)
248 {
249 void *mc_outer = MLX5_ADDR_OF(fte_match_param, match_criteria,
250 outer_headers);
251 void *mc_misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
252 misc_parameters);
253 void *mc_inner = MLX5_ADDR_OF(fte_match_param, match_criteria,
254 inner_headers);
255 int mc_outer_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
256 int mc_misc_sz = MLX5_ST_SZ_BYTES(fte_match_set_misc);
257 int mc_inner_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
258 int i;
259
260 for (i = 0; i < ft->num_groups; i++) {
261 struct mlx5_flow_table_group *g = &ft->group[i].g;
262 void *gmc_outer = MLX5_ADDR_OF(fte_match_param,
263 g->match_criteria,
264 outer_headers);
265 void *gmc_misc = MLX5_ADDR_OF(fte_match_param,
266 g->match_criteria,
267 misc_parameters);
268 void *gmc_inner = MLX5_ADDR_OF(fte_match_param,
269 g->match_criteria,
270 inner_headers);
271
272 if (g->match_criteria_enable != match_criteria_enable)
273 continue;
274
275 if (match_criteria_enable & MLX5_MATCH_OUTER_HEADERS)
276 if (memcmp(mc_outer, gmc_outer, mc_outer_sz))
277 continue;
278
279 if (match_criteria_enable & MLX5_MATCH_MISC_PARAMETERS)
280 if (memcmp(mc_misc, gmc_misc, mc_misc_sz))
281 continue;
282
283 if (match_criteria_enable & MLX5_MATCH_INNER_HEADERS)
284 if (memcmp(mc_inner, gmc_inner, mc_inner_sz))
285 continue;
286
287 *group_ix = i;
288 return 0;
289 }
290
291 return -EINVAL;
292 }
293
alloc_flow_index(struct mlx5_flow_table * ft,int group_ix,u32 * ix)294 static int alloc_flow_index(struct mlx5_flow_table *ft, int group_ix, u32 *ix)
295 {
296 struct mlx5_ftg *g = &ft->group[group_ix];
297 int err = 0;
298
299 mutex_lock(&ft->mutex);
300
301 *ix = find_next_zero_bit(ft->bitmap, ft->size, g->start_ix);
302 if (*ix >= (g->start_ix + (1 << g->g.log_sz)))
303 err = -ENOSPC;
304 else
305 __set_bit(*ix, ft->bitmap);
306
307 mutex_unlock(&ft->mutex);
308
309 return err;
310 }
311
mlx5_free_flow_index(struct mlx5_flow_table * ft,u32 ix)312 static void mlx5_free_flow_index(struct mlx5_flow_table *ft, u32 ix)
313 {
314 __clear_bit(ix, ft->bitmap);
315 }
316
mlx5_add_flow_table_entry(void * flow_table,u8 match_criteria_enable,void * match_criteria,void * flow_context,u32 * flow_index)317 int mlx5_add_flow_table_entry(void *flow_table, u8 match_criteria_enable,
318 void *match_criteria, void *flow_context,
319 u32 *flow_index)
320 {
321 struct mlx5_flow_table *ft = flow_table;
322 int group_ix;
323 int err;
324
325 err = mlx5_find_group(ft, match_criteria_enable, match_criteria,
326 &group_ix);
327 if (err) {
328 mlx5_core_warn(ft->dev, "mlx5_find_group failed\n");
329 return err;
330 }
331
332 err = alloc_flow_index(ft, group_ix, flow_index);
333 if (err) {
334 mlx5_core_warn(ft->dev, "alloc_flow_index failed\n");
335 return err;
336 }
337
338 err = mlx5_set_flow_entry_cmd(ft, group_ix, *flow_index, flow_context);
339 if (err)
340 mlx5_free_flow_index(ft, *flow_index);
341
342 return err;
343 }
344 EXPORT_SYMBOL(mlx5_add_flow_table_entry);
345
mlx5_del_flow_table_entry(void * flow_table,u32 flow_index)346 void mlx5_del_flow_table_entry(void *flow_table, u32 flow_index)
347 {
348 struct mlx5_flow_table *ft = flow_table;
349
350 mlx5_del_flow_entry_cmd(ft, flow_index);
351 mlx5_free_flow_index(ft, flow_index);
352 }
353 EXPORT_SYMBOL(mlx5_del_flow_table_entry);
354
mlx5_create_flow_table(struct mlx5_core_dev * dev,u8 level,u8 table_type,u16 vport,u16 num_groups,struct mlx5_flow_table_group * group)355 void *mlx5_create_flow_table(struct mlx5_core_dev *dev, u8 level, u8 table_type,
356 u16 vport,
357 u16 num_groups,
358 struct mlx5_flow_table_group *group)
359 {
360 struct mlx5_flow_table *ft;
361 u32 start_ix = 0;
362 u32 ft_size = 0;
363 void *gr;
364 void *bm;
365 int err;
366 int i;
367
368 for (i = 0; i < num_groups; i++)
369 ft_size += (1 << group[i].log_sz);
370
371 ft = kzalloc(sizeof(*ft), GFP_KERNEL);
372 gr = kcalloc(num_groups, sizeof(struct mlx5_ftg), GFP_KERNEL);
373 bm = kcalloc(BITS_TO_LONGS(ft_size), sizeof(uintptr_t), GFP_KERNEL);
374
375 ft->group = gr;
376 ft->bitmap = bm;
377 ft->num_groups = num_groups;
378 ft->level = level;
379 ft->vport = vport;
380 ft->type = table_type;
381 ft->size = ft_size;
382 ft->dev = dev;
383 mutex_init(&ft->mutex);
384
385 for (i = 0; i < ft->num_groups; i++) {
386 memcpy(&ft->group[i].g, &group[i], sizeof(*group));
387 ft->group[i].start_ix = start_ix;
388 start_ix += 1 << group[i].log_sz;
389 }
390
391 err = mlx5_create_flow_table_cmd(ft);
392 if (err)
393 goto err_free_ft;
394
395 err = mlx5_create_flow_table_groups(ft);
396 if (err)
397 goto err_destroy_flow_table_cmd;
398
399 return ft;
400
401 err_destroy_flow_table_cmd:
402 mlx5_destroy_flow_table_cmd(ft);
403
404 err_free_ft:
405 mlx5_core_warn(dev, "failed to alloc flow table\n");
406 kfree(bm);
407 kfree(gr);
408 kfree(ft);
409
410 return NULL;
411 }
412 EXPORT_SYMBOL(mlx5_create_flow_table);
413
mlx5_destroy_flow_table(void * flow_table)414 void mlx5_destroy_flow_table(void *flow_table)
415 {
416 struct mlx5_flow_table *ft = flow_table;
417
418 mlx5_destroy_flow_table_groups(ft);
419 mlx5_destroy_flow_table_cmd(ft);
420 kfree(ft->bitmap);
421 kfree(ft->group);
422 kfree(ft);
423 }
424 EXPORT_SYMBOL(mlx5_destroy_flow_table);
425
mlx5_get_flow_table_id(void * flow_table)426 u32 mlx5_get_flow_table_id(void *flow_table)
427 {
428 struct mlx5_flow_table *ft = flow_table;
429
430 return ft->id;
431 }
432 EXPORT_SYMBOL(mlx5_get_flow_table_id);
433