1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3 *
4 * Copyright (c) 2017 - 2026 Intel Corporation
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenFabrics.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "osdep.h"
36 #include "irdma_hmc.h"
37 #include "irdma_defs.h"
38 #include "irdma_type.h"
39 #include "irdma_protos.h"
40
41 #include "irdma_ws.h"
42
43 /**
44 * irdma_alloc_node - Allocate a WS node and init
45 * @vsi: vsi pointer
46 * @user_pri: user priority
47 * @node_type: Type of node, leaf or parent
48 * @parent: parent node pointer
49 */
50 static struct irdma_ws_node *
irdma_alloc_node(struct irdma_sc_vsi * vsi,u8 user_pri,enum irdma_ws_node_type node_type,struct irdma_ws_node * parent)51 irdma_alloc_node(struct irdma_sc_vsi *vsi,
52 u8 user_pri,
53 enum irdma_ws_node_type node_type,
54 struct irdma_ws_node *parent)
55 {
56 struct irdma_virt_mem ws_mem;
57 struct irdma_ws_node *node;
58 u16 node_index = 0;
59
60 ws_mem.size = sizeof(*node);
61 ws_mem.va = kzalloc(ws_mem.size, GFP_KERNEL);
62 if (!ws_mem.va)
63 return NULL;
64
65 if (parent) {
66 node_index = irdma_alloc_ws_node_id(vsi->dev);
67 if (node_index == IRDMA_WS_NODE_INVALID) {
68 kfree(ws_mem.va);
69 return NULL;
70 }
71 }
72
73 node = ws_mem.va;
74 node->index = node_index;
75 node->vsi_index = vsi->vsi_idx;
76 INIT_LIST_HEAD(&node->child_list_head);
77 if (node_type == WS_NODE_TYPE_LEAF) {
78 node->type_leaf = true;
79 node->traffic_class = vsi->qos[user_pri].traffic_class;
80 node->user_pri = user_pri;
81 node->rel_bw = vsi->qos[user_pri].rel_bw;
82 if (!node->rel_bw)
83 node->rel_bw = 1;
84
85 node->prio_type = IRDMA_PRIO_WEIGHTED_RR;
86 } else {
87 node->rel_bw = 1;
88 node->prio_type = IRDMA_PRIO_WEIGHTED_RR;
89 node->enable = true;
90 }
91
92 node->parent = parent;
93
94 return node;
95 }
96
97 /**
98 * irdma_free_node - Free a WS node
99 * @vsi: VSI stricture of device
100 * @node: Pointer to node to free
101 */
102 static void
irdma_free_node(struct irdma_sc_vsi * vsi,struct irdma_ws_node * node)103 irdma_free_node(struct irdma_sc_vsi *vsi,
104 struct irdma_ws_node *node)
105 {
106 struct irdma_virt_mem ws_mem;
107
108 if (node->index)
109 irdma_free_ws_node_id(vsi->dev, node->index);
110
111 ws_mem.va = node;
112 ws_mem.size = sizeof(*node);
113 kfree(ws_mem.va);
114 }
115
116 /**
117 * irdma_ws_cqp_cmd - Post CQP work scheduler node cmd
118 * @vsi: vsi pointer
119 * @node: pointer to node
120 * @cmd: add, remove or modify
121 * @qs_handle: Pointer to store the qs_handle for a leaf node
122 */
123 static int
irdma_ws_cqp_cmd(struct irdma_sc_vsi * vsi,struct irdma_ws_node * node,u8 cmd,u16 * qs_handle)124 irdma_ws_cqp_cmd(struct irdma_sc_vsi *vsi,
125 struct irdma_ws_node *node, u8 cmd, u16 *qs_handle)
126 {
127 struct irdma_ws_node_info node_info = {0};
128
129 node_info.id = node->index;
130 node_info.vsi = node->vsi_index;
131 if (node->parent)
132 node_info.parent_id = node->parent->index;
133 else
134 node_info.parent_id = node_info.id;
135
136 node_info.weight = node->rel_bw;
137 node_info.tc = node->traffic_class;
138 node_info.prio_type = node->prio_type;
139 node_info.type_leaf = node->type_leaf;
140 node_info.enable = node->enable;
141 if (irdma_cqp_ws_node_cmd(vsi->dev, cmd, &node_info)) {
142 irdma_debug(vsi->dev, IRDMA_DEBUG_WS, "CQP WS CMD failed\n");
143 return -ENOMEM;
144 }
145
146 if (node->type_leaf && cmd == IRDMA_OP_WS_ADD_NODE && qs_handle)
147 *qs_handle = node_info.qs_handle;
148
149 return 0;
150 }
151
152 /**
153 * ws_find_node - Find SC WS node based on VSI id or TC
154 * @parent: parent node of First VSI or TC node
155 * @match_val: value to match
156 * @type: match type VSI/TC
157 */
158 static struct irdma_ws_node *
ws_find_node(struct irdma_ws_node * parent,u16 match_val,enum irdma_ws_match_type type)159 ws_find_node(struct irdma_ws_node *parent,
160 u16 match_val,
161 enum irdma_ws_match_type type)
162 {
163 struct irdma_ws_node *node;
164
165 switch (type) {
166 case WS_MATCH_TYPE_VSI:
167 list_for_each_entry(node, &parent->child_list_head, siblings) {
168 if (node->vsi_index == match_val)
169 return node;
170 }
171 break;
172 case WS_MATCH_TYPE_TC:
173 list_for_each_entry(node, &parent->child_list_head, siblings) {
174 if (node->traffic_class == match_val)
175 return node;
176 }
177 break;
178 default:
179 break;
180 }
181
182 return NULL;
183 }
184
185 /**
186 * irdma_ws_in_use - Checks to see if a leaf node is in use
187 * @vsi: vsi pointer
188 * @user_pri: user priority
189 */
190 static bool
irdma_ws_in_use(struct irdma_sc_vsi * vsi,u8 user_pri)191 irdma_ws_in_use(struct irdma_sc_vsi *vsi, u8 user_pri)
192 {
193 int i;
194
195 if (!list_empty(&vsi->qos[user_pri].qplist))
196 return true;
197
198 /*
199 * Check if the qs handle associated with the given user priority is in use by any other user priority. If so,
200 * nothing left to do
201 */
202 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
203 if (vsi->qos[i].qs_handle == vsi->qos[user_pri].qs_handle &&
204 !list_empty(&vsi->qos[i].qplist))
205 return true;
206 }
207
208 return false;
209 }
210
211 /**
212 * irdma_remove_leaf - Remove leaf node unconditionally
213 * @vsi: vsi pointer
214 * @user_pri: user priority
215 */
216 static void
irdma_remove_leaf(struct irdma_sc_vsi * vsi,u8 user_pri)217 irdma_remove_leaf(struct irdma_sc_vsi *vsi, u8 user_pri)
218 {
219 struct irdma_ws_node *ws_tree_root, *vsi_node, *tc_node;
220 u16 qs_handle;
221 int i;
222
223 qs_handle = vsi->qos[user_pri].qs_handle;
224 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
225 if (vsi->qos[i].qs_handle == qs_handle)
226 vsi->qos[i].valid = false;
227 }
228
229 ws_tree_root = vsi->dev->ws_tree_root;
230 if (!ws_tree_root)
231 return;
232
233 vsi_node = ws_find_node(ws_tree_root, vsi->vsi_idx,
234 WS_MATCH_TYPE_VSI);
235 if (!vsi_node)
236 return;
237
238 tc_node = ws_find_node(vsi_node,
239 vsi->qos[user_pri].traffic_class,
240 WS_MATCH_TYPE_TC);
241 if (!tc_node)
242 return;
243
244 list_del(&tc_node->siblings);
245 irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_DELETE_NODE, NULL);
246
247 vsi->unregister_qset(vsi, tc_node);
248 irdma_free_node(vsi, tc_node);
249 /* Check if VSI node can be freed */
250 if (list_empty(&vsi_node->child_list_head)) {
251 irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE, NULL);
252 list_del(&vsi_node->siblings);
253 irdma_free_node(vsi, vsi_node);
254 /* Free head node there are no remaining VSI nodes */
255 if (list_empty(&ws_tree_root->child_list_head)) {
256 irdma_ws_cqp_cmd(vsi, ws_tree_root,
257 IRDMA_OP_WS_DELETE_NODE, NULL);
258 irdma_free_node(vsi, ws_tree_root);
259 vsi->dev->ws_tree_root = NULL;
260 }
261 }
262 }
263
264 static int
irdma_enable_leaf(struct irdma_sc_vsi * vsi,struct irdma_ws_node * tc_node)265 irdma_enable_leaf(struct irdma_sc_vsi *vsi,
266 struct irdma_ws_node *tc_node)
267 {
268 int ret;
269
270 ret = vsi->register_qset(vsi, tc_node);
271 if (ret)
272 return ret;
273
274 tc_node->enable = true;
275 ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_MODIFY_NODE, NULL);
276 if (ret)
277 goto enable_err;
278 return 0;
279
280 enable_err:
281 vsi->unregister_qset(vsi, tc_node);
282
283 return ret;
284 }
285
286 static struct irdma_ws_node *
irdma_add_leaf_node(struct irdma_sc_vsi * vsi,struct irdma_ws_node * vsi_node,u8 user_pri,u16 traffic_class)287 irdma_add_leaf_node(struct irdma_sc_vsi *vsi,
288 struct irdma_ws_node *vsi_node,
289 u8 user_pri, u16 traffic_class)
290 {
291 struct irdma_ws_node *tc_node =
292 irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_LEAF, vsi_node);
293 int i, ret = 0;
294
295 if (!tc_node)
296 return NULL;
297 ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_ADD_NODE, &tc_node->qs_handle);
298 if (ret) {
299 irdma_free_node(vsi, tc_node);
300 return NULL;
301 }
302 vsi->qos[tc_node->user_pri].qs_handle = tc_node->qs_handle;
303
304 list_add(&tc_node->siblings, &vsi_node->child_list_head);
305
306 ret = irdma_enable_leaf(vsi, tc_node);
307 if (ret)
308 goto reg_err;
309
310 /*
311 * Iterate through other UPs and update the QS handle if they have a matching traffic class.
312 */
313 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
314 if (vsi->qos[i].traffic_class == traffic_class) {
315 vsi->qos[i].qs_handle = tc_node->qs_handle;
316 vsi->qos[i].l2_sched_node_id =
317 tc_node->l2_sched_node_id;
318 vsi->qos[i].valid = true;
319 }
320 }
321 return tc_node;
322
323 reg_err:
324 irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_DELETE_NODE, NULL);
325 list_del(&tc_node->siblings);
326 irdma_free_node(vsi, tc_node);
327
328 return NULL;
329 }
330
331 /**
332 * irdma_ws_add - Build work scheduler tree, set RDMA qs_handle
333 * @vsi: vsi pointer
334 * @user_pri: user priority
335 */
336 int
irdma_ws_add(struct irdma_sc_vsi * vsi,u8 user_pri)337 irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
338 {
339 struct irdma_ws_node *ws_tree_root;
340 struct irdma_ws_node *vsi_node;
341 struct irdma_ws_node *tc_node;
342 u16 traffic_class;
343 int ret = 0;
344
345 mutex_lock(&vsi->dev->ws_mutex);
346 if (vsi->tc_change_pending) {
347 ret = -EBUSY;
348 goto exit;
349 }
350
351 if (vsi->qos[user_pri].valid)
352 goto exit;
353
354 ws_tree_root = vsi->dev->ws_tree_root;
355 if (!ws_tree_root) {
356 ws_tree_root = irdma_alloc_node(vsi, user_pri,
357 WS_NODE_TYPE_PARENT, NULL);
358 if (!ws_tree_root) {
359 ret = -ENOMEM;
360 goto exit;
361 }
362 irdma_debug(vsi->dev, IRDMA_DEBUG_WS,
363 "Creating root node = %d\n", ws_tree_root->index);
364
365 ret = irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_ADD_NODE,
366 NULL);
367 if (ret) {
368 irdma_free_node(vsi, ws_tree_root);
369 goto exit;
370 }
371
372 vsi->dev->ws_tree_root = ws_tree_root;
373 }
374
375 /* Find a second tier node that matches the VSI */
376 vsi_node = ws_find_node(ws_tree_root, vsi->vsi_idx,
377 WS_MATCH_TYPE_VSI);
378
379 /* If VSI node doesn't exist, add one */
380 if (!vsi_node) {
381 irdma_debug(vsi->dev, IRDMA_DEBUG_WS,
382 "Node not found matching VSI %d\n", vsi->vsi_idx);
383 vsi_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_PARENT,
384 ws_tree_root);
385 if (!vsi_node) {
386 ret = -ENOMEM;
387 goto vsi_add_err;
388 }
389
390 ret = irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_ADD_NODE,
391 NULL);
392 if (ret) {
393 irdma_free_node(vsi, vsi_node);
394 goto vsi_add_err;
395 }
396
397 list_add(&vsi_node->siblings, &ws_tree_root->child_list_head);
398 }
399
400 irdma_debug(vsi->dev, IRDMA_DEBUG_WS,
401 "Using node %d which represents VSI %d\n", vsi_node->index,
402 vsi->vsi_idx);
403 traffic_class = vsi->qos[user_pri].traffic_class;
404 tc_node = ws_find_node(vsi_node, traffic_class,
405 WS_MATCH_TYPE_TC);
406 if (!tc_node) {
407 /* Add leaf node */
408 irdma_debug(vsi->dev, IRDMA_DEBUG_WS,
409 "Node not found matching VSI %d and TC %d\n",
410 vsi->vsi_idx, traffic_class);
411 tc_node = irdma_add_leaf_node(vsi, vsi_node, user_pri,
412 traffic_class);
413 if (!tc_node) {
414 ret = -ENOMEM;
415 goto leaf_add_err;
416 }
417 }
418 irdma_debug(vsi->dev, IRDMA_DEBUG_WS,
419 "Using node %d which represents VSI %d TC %d\n",
420 tc_node->index, vsi->vsi_idx, traffic_class);
421 goto exit;
422
423 leaf_add_err:
424 if (list_empty(&vsi_node->child_list_head)) {
425 if (irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE,
426 NULL))
427 goto exit;
428 list_del(&vsi_node->siblings);
429 irdma_free_node(vsi, vsi_node);
430 }
431
432 vsi_add_err:
433 /* Free head node there are no remaining VSI nodes */
434 if (list_empty(&ws_tree_root->child_list_head)) {
435 irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_DELETE_NODE,
436 NULL);
437 vsi->dev->ws_tree_root = NULL;
438 irdma_free_node(vsi, ws_tree_root);
439 }
440
441 exit:
442 mutex_unlock(&vsi->dev->ws_mutex);
443 return ret;
444 }
445
446 /**
447 * irdma_ws_remove - Free WS scheduler node, update WS tree
448 * @vsi: vsi pointer
449 * @user_pri: user priority
450 */
451 void
irdma_ws_remove(struct irdma_sc_vsi * vsi,u8 user_pri)452 irdma_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)
453 {
454 mutex_lock(&vsi->qos[user_pri].qos_mutex);
455 mutex_lock(&vsi->dev->ws_mutex);
456 if (irdma_ws_in_use(vsi, user_pri))
457 goto exit;
458 irdma_remove_leaf(vsi, user_pri);
459 exit:
460 mutex_unlock(&vsi->dev->ws_mutex);
461 mutex_unlock(&vsi->qos[user_pri].qos_mutex);
462 }
463
464 /**
465 * irdma_ws_reset - Reset entire WS tree
466 * @vsi: vsi pointer
467 */
468 void
irdma_ws_reset(struct irdma_sc_vsi * vsi)469 irdma_ws_reset(struct irdma_sc_vsi *vsi)
470 {
471 u8 i;
472
473 mutex_lock(&vsi->dev->ws_mutex);
474 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; ++i)
475 irdma_remove_leaf(vsi, i);
476 mutex_unlock(&vsi->dev->ws_mutex);
477 }
478