1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25 #include <linux/ratelimit.h>
26 #include <linux/printk.h>
27 #include <linux/slab.h>
28 #include <linux/list.h>
29 #include <linux/types.h>
30 #include <linux/bitops.h>
31 #include <linux/sched.h>
32 #include "kfd_priv.h"
33 #include "kfd_device_queue_manager.h"
34 #include "kfd_mqd_manager.h"
35 #include "cik_regs.h"
36 #include "kfd_kernel_queue.h"
37 #include "amdgpu_amdkfd.h"
38 #include "amdgpu_reset.h"
39 #include "amdgpu_sdma.h"
40 #include "mes_v11_api_def.h"
41 #include "kfd_debug.h"
42
43 /* Size of the per-pipe EOP queue */
44 #define CIK_HPD_EOP_BYTES_LOG2 11
45 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
46 /* See unmap_queues_cpsch() */
47 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff
48
49 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
50 u32 pasid, unsigned int vmid);
51
52 static int execute_queues_cpsch(struct device_queue_manager *dqm,
53 enum kfd_unmap_queues_filter filter,
54 uint32_t filter_param,
55 uint32_t grace_period);
56 static int unmap_queues_cpsch(struct device_queue_manager *dqm,
57 enum kfd_unmap_queues_filter filter,
58 uint32_t filter_param,
59 uint32_t grace_period,
60 bool reset);
61
62 static int map_queues_cpsch(struct device_queue_manager *dqm);
63
64 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
65 struct queue *q);
66
67 static inline void deallocate_hqd(struct device_queue_manager *dqm,
68 struct queue *q);
69 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
70 static int allocate_sdma_queue(struct device_queue_manager *dqm,
71 struct queue *q, const uint32_t *restore_sdma_id);
72
73 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
74
75 static inline
get_mqd_type_from_queue_type(enum kfd_queue_type type)76 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
77 {
78 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
79 return KFD_MQD_TYPE_SDMA;
80 return KFD_MQD_TYPE_CP;
81 }
82
is_pipe_enabled(struct device_queue_manager * dqm,int mec,int pipe)83 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
84 {
85 int i;
86 int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec
87 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe;
88
89 /* queue is available for KFD usage if bit is 1 */
90 for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i)
91 if (test_bit(pipe_offset + i,
92 dqm->dev->kfd->shared_resources.cp_queue_bitmap))
93 return true;
94 return false;
95 }
96
get_cp_queues_num(struct device_queue_manager * dqm)97 unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
98 {
99 return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap,
100 AMDGPU_MAX_QUEUES);
101 }
102
get_queues_per_pipe(struct device_queue_manager * dqm)103 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
104 {
105 return dqm->dev->kfd->shared_resources.num_queue_per_pipe;
106 }
107
get_pipes_per_mec(struct device_queue_manager * dqm)108 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
109 {
110 return dqm->dev->kfd->shared_resources.num_pipe_per_mec;
111 }
112
get_num_all_sdma_engines(struct device_queue_manager * dqm)113 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
114 {
115 return kfd_get_num_sdma_engines(dqm->dev) +
116 kfd_get_num_xgmi_sdma_engines(dqm->dev);
117 }
118
get_num_sdma_queues(struct device_queue_manager * dqm)119 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
120 {
121 return kfd_get_num_sdma_engines(dqm->dev) *
122 dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
123 }
124
get_num_xgmi_sdma_queues(struct device_queue_manager * dqm)125 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
126 {
127 return kfd_get_num_xgmi_sdma_engines(dqm->dev) *
128 dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
129 }
130
init_sdma_bitmaps(struct device_queue_manager * dqm)131 static void init_sdma_bitmaps(struct device_queue_manager *dqm)
132 {
133 bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES);
134 bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm));
135
136 bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
137 bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm));
138
139 /* Mask out the reserved queues */
140 bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap,
141 dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap,
142 KFD_MAX_SDMA_QUEUES);
143 }
144
program_sh_mem_settings(struct device_queue_manager * dqm,struct qcm_process_device * qpd)145 void program_sh_mem_settings(struct device_queue_manager *dqm,
146 struct qcm_process_device *qpd)
147 {
148 uint32_t xcc_mask = dqm->dev->xcc_mask;
149 int xcc_id;
150
151 for_each_inst(xcc_id, xcc_mask)
152 dqm->dev->kfd2kgd->program_sh_mem_settings(
153 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config,
154 qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit,
155 qpd->sh_mem_bases, xcc_id);
156 }
157
kfd_hws_hang(struct device_queue_manager * dqm)158 static void kfd_hws_hang(struct device_queue_manager *dqm)
159 {
160 struct device_process_node *cur;
161 struct qcm_process_device *qpd;
162 struct queue *q;
163
164 /* Mark all device queues as reset. */
165 list_for_each_entry(cur, &dqm->queues, list) {
166 qpd = cur->qpd;
167 list_for_each_entry(q, &qpd->queues_list, list) {
168 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
169
170 pdd->has_reset_queue = true;
171 }
172 }
173
174 /*
175 * Issue a GPU reset if HWS is unresponsive
176 */
177 amdgpu_amdkfd_gpu_reset(dqm->dev->adev);
178 }
179
convert_to_mes_queue_type(int queue_type)180 static int convert_to_mes_queue_type(int queue_type)
181 {
182 int mes_queue_type;
183
184 switch (queue_type) {
185 case KFD_QUEUE_TYPE_COMPUTE:
186 mes_queue_type = MES_QUEUE_TYPE_COMPUTE;
187 break;
188 case KFD_QUEUE_TYPE_SDMA:
189 mes_queue_type = MES_QUEUE_TYPE_SDMA;
190 break;
191 default:
192 WARN(1, "Invalid queue type %d", queue_type);
193 mes_queue_type = -EINVAL;
194 break;
195 }
196
197 return mes_queue_type;
198 }
199
add_queue_mes(struct device_queue_manager * dqm,struct queue * q,struct qcm_process_device * qpd)200 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
201 struct qcm_process_device *qpd)
202 {
203 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
204 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
205 struct mes_add_queue_input queue_input;
206 int r, queue_type;
207 uint64_t wptr_addr_off;
208
209 if (!dqm->sched_running || dqm->sched_halt)
210 return 0;
211 if (!down_read_trylock(&adev->reset_domain->sem))
212 return -EIO;
213
214 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
215 queue_input.process_id = pdd->pasid;
216 queue_input.page_table_base_addr = qpd->page_table_base;
217 queue_input.process_va_start = 0;
218 queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
219 /* MES unit for quantum is 100ns */
220 queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */
221 queue_input.process_context_addr = pdd->proc_ctx_gpu_addr;
222 queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */
223 queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
224 queue_input.inprocess_gang_priority = q->properties.priority;
225 queue_input.gang_global_priority_level =
226 AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
227 queue_input.doorbell_offset = q->properties.doorbell_off;
228 queue_input.mqd_addr = q->gart_mqd_addr;
229 queue_input.wptr_addr = (uint64_t)q->properties.write_ptr;
230
231 wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1);
232 queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off;
233
234 queue_input.is_kfd_process = 1;
235 queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);
236 queue_input.queue_size = q->properties.queue_size >> 2;
237
238 queue_input.paging = false;
239 queue_input.tba_addr = qpd->tba_addr;
240 queue_input.tma_addr = qpd->tma_addr;
241 queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device);
242 queue_input.skip_process_ctx_clear =
243 qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED &&
244 (qpd->pqm->process->debug_trap_enabled ||
245 kfd_dbg_has_ttmps_always_setup(q->device));
246
247 queue_type = convert_to_mes_queue_type(q->properties.type);
248 if (queue_type < 0) {
249 dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n",
250 q->properties.type);
251 up_read(&adev->reset_domain->sem);
252 return -EINVAL;
253 }
254 queue_input.queue_type = (uint32_t)queue_type;
255
256 queue_input.exclusively_scheduled = q->properties.is_gws;
257
258 amdgpu_mes_lock(&adev->mes);
259 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
260 amdgpu_mes_unlock(&adev->mes);
261 up_read(&adev->reset_domain->sem);
262 if (r) {
263 dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n",
264 q->properties.doorbell_off);
265 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
266 kfd_hws_hang(dqm);
267 }
268
269 return r;
270 }
271
remove_queue_mes(struct device_queue_manager * dqm,struct queue * q,struct qcm_process_device * qpd)272 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
273 struct qcm_process_device *qpd)
274 {
275 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
276 int r;
277 struct mes_remove_queue_input queue_input;
278
279 if (!dqm->sched_running || dqm->sched_halt)
280 return 0;
281 if (!down_read_trylock(&adev->reset_domain->sem))
282 return -EIO;
283
284 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
285 queue_input.doorbell_offset = q->properties.doorbell_off;
286 queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
287
288 amdgpu_mes_lock(&adev->mes);
289 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
290 amdgpu_mes_unlock(&adev->mes);
291 up_read(&adev->reset_domain->sem);
292
293 if (r) {
294 dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n",
295 q->properties.doorbell_off);
296 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
297 kfd_hws_hang(dqm);
298 }
299
300 return r;
301 }
302
remove_all_kfd_queues_mes(struct device_queue_manager * dqm)303 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
304 {
305 struct device_process_node *cur;
306 struct device *dev = dqm->dev->adev->dev;
307 struct qcm_process_device *qpd;
308 struct queue *q;
309 int retval = 0;
310
311 list_for_each_entry(cur, &dqm->queues, list) {
312 qpd = cur->qpd;
313 list_for_each_entry(q, &qpd->queues_list, list) {
314 if (q->properties.is_active) {
315 retval = remove_queue_mes(dqm, q, qpd);
316 if (retval) {
317 dev_err(dev, "%s: Failed to remove queue %d for dev %d",
318 __func__,
319 q->properties.queue_id,
320 dqm->dev->id);
321 return retval;
322 }
323 }
324 }
325 }
326
327 return retval;
328 }
329
add_all_kfd_queues_mes(struct device_queue_manager * dqm)330 static int add_all_kfd_queues_mes(struct device_queue_manager *dqm)
331 {
332 struct device_process_node *cur;
333 struct device *dev = dqm->dev->adev->dev;
334 struct qcm_process_device *qpd;
335 struct queue *q;
336 int retval = 0;
337
338 list_for_each_entry(cur, &dqm->queues, list) {
339 qpd = cur->qpd;
340 list_for_each_entry(q, &qpd->queues_list, list) {
341 if (!q->properties.is_active)
342 continue;
343 retval = add_queue_mes(dqm, q, qpd);
344 if (retval) {
345 dev_err(dev, "%s: Failed to add queue %d for dev %d",
346 __func__,
347 q->properties.queue_id,
348 dqm->dev->id);
349 return retval;
350 }
351 }
352 }
353
354 return retval;
355 }
356
suspend_all_queues_mes(struct device_queue_manager * dqm)357 static int suspend_all_queues_mes(struct device_queue_manager *dqm)
358 {
359 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
360 int r = 0;
361
362 if (!down_read_trylock(&adev->reset_domain->sem))
363 return -EIO;
364
365 r = amdgpu_mes_suspend(adev);
366 up_read(&adev->reset_domain->sem);
367
368 if (r) {
369 dev_err(adev->dev, "failed to suspend gangs from MES\n");
370 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
371 kfd_hws_hang(dqm);
372 }
373
374 return r;
375 }
376
resume_all_queues_mes(struct device_queue_manager * dqm)377 static int resume_all_queues_mes(struct device_queue_manager *dqm)
378 {
379 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
380 int r = 0;
381
382 if (!down_read_trylock(&adev->reset_domain->sem))
383 return -EIO;
384
385 r = amdgpu_mes_resume(adev);
386 up_read(&adev->reset_domain->sem);
387
388 if (r) {
389 dev_err(adev->dev, "failed to resume gangs from MES\n");
390 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
391 kfd_hws_hang(dqm);
392 }
393
394 return r;
395 }
396
increment_queue_count(struct device_queue_manager * dqm,struct qcm_process_device * qpd,struct queue * q)397 static void increment_queue_count(struct device_queue_manager *dqm,
398 struct qcm_process_device *qpd,
399 struct queue *q)
400 {
401 dqm->active_queue_count++;
402 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
403 q->properties.type == KFD_QUEUE_TYPE_DIQ)
404 dqm->active_cp_queue_count++;
405
406 if (q->properties.is_gws) {
407 dqm->gws_queue_count++;
408 qpd->mapped_gws_queue = true;
409 }
410 }
411
decrement_queue_count(struct device_queue_manager * dqm,struct qcm_process_device * qpd,struct queue * q)412 static void decrement_queue_count(struct device_queue_manager *dqm,
413 struct qcm_process_device *qpd,
414 struct queue *q)
415 {
416 dqm->active_queue_count--;
417 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
418 q->properties.type == KFD_QUEUE_TYPE_DIQ)
419 dqm->active_cp_queue_count--;
420
421 if (q->properties.is_gws) {
422 dqm->gws_queue_count--;
423 qpd->mapped_gws_queue = false;
424 }
425 }
426
427 /*
428 * Allocate a doorbell ID to this queue.
429 * If doorbell_id is passed in, make sure requested ID is valid then allocate it.
430 */
allocate_doorbell(struct qcm_process_device * qpd,struct queue * q,uint32_t const * restore_id)431 static int allocate_doorbell(struct qcm_process_device *qpd,
432 struct queue *q,
433 uint32_t const *restore_id)
434 {
435 struct kfd_node *dev = qpd->dqm->dev;
436
437 if (!KFD_IS_SOC15(dev)) {
438 /* On pre-SOC15 chips we need to use the queue ID to
439 * preserve the user mode ABI.
440 */
441
442 if (restore_id && *restore_id != q->properties.queue_id)
443 return -EINVAL;
444
445 q->doorbell_id = q->properties.queue_id;
446 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
447 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
448 /* For SDMA queues on SOC15 with 8-byte doorbell, use static
449 * doorbell assignments based on the engine and queue id.
450 * The doobell index distance between RLC (2*i) and (2*i+1)
451 * for a SDMA engine is 512.
452 */
453
454 uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx;
455
456 /*
457 * q->properties.sdma_engine_id corresponds to the virtual
458 * sdma engine number. However, for doorbell allocation,
459 * we need the physical sdma engine id in order to get the
460 * correct doorbell offset.
461 */
462 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id *
463 get_num_all_sdma_engines(qpd->dqm) +
464 q->properties.sdma_engine_id]
465 + (q->properties.sdma_queue_id & 1)
466 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET
467 + (q->properties.sdma_queue_id >> 1);
468
469 if (restore_id && *restore_id != valid_id)
470 return -EINVAL;
471 q->doorbell_id = valid_id;
472 } else {
473 /* For CP queues on SOC15 */
474 if (restore_id) {
475 /* make sure that ID is free */
476 if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))
477 return -EINVAL;
478
479 q->doorbell_id = *restore_id;
480 } else {
481 /* or reserve a free doorbell ID */
482 unsigned int found;
483
484 found = find_first_zero_bit(qpd->doorbell_bitmap,
485 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
486 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
487 pr_debug("No doorbells available");
488 return -EBUSY;
489 }
490 set_bit(found, qpd->doorbell_bitmap);
491 q->doorbell_id = found;
492 }
493 }
494
495 q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev,
496 qpd->proc_doorbells,
497 q->doorbell_id,
498 dev->kfd->device_info.doorbell_size);
499 return 0;
500 }
501
deallocate_doorbell(struct qcm_process_device * qpd,struct queue * q)502 static void deallocate_doorbell(struct qcm_process_device *qpd,
503 struct queue *q)
504 {
505 unsigned int old;
506 struct kfd_node *dev = qpd->dqm->dev;
507
508 if (!KFD_IS_SOC15(dev) ||
509 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
510 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
511 return;
512
513 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
514 WARN_ON(!old);
515 }
516
program_trap_handler_settings(struct device_queue_manager * dqm,struct qcm_process_device * qpd)517 static void program_trap_handler_settings(struct device_queue_manager *dqm,
518 struct qcm_process_device *qpd)
519 {
520 uint32_t xcc_mask = dqm->dev->xcc_mask;
521 int xcc_id;
522
523 if (dqm->dev->kfd2kgd->program_trap_handler_settings)
524 for_each_inst(xcc_id, xcc_mask)
525 dqm->dev->kfd2kgd->program_trap_handler_settings(
526 dqm->dev->adev, qpd->vmid, qpd->tba_addr,
527 qpd->tma_addr, xcc_id);
528 }
529
allocate_vmid(struct device_queue_manager * dqm,struct qcm_process_device * qpd,struct queue * q)530 static int allocate_vmid(struct device_queue_manager *dqm,
531 struct qcm_process_device *qpd,
532 struct queue *q)
533 {
534 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
535 struct device *dev = dqm->dev->adev->dev;
536 int allocated_vmid = -1, i;
537
538 for (i = dqm->dev->vm_info.first_vmid_kfd;
539 i <= dqm->dev->vm_info.last_vmid_kfd; i++) {
540 if (!dqm->vmid_pasid[i]) {
541 allocated_vmid = i;
542 break;
543 }
544 }
545
546 if (allocated_vmid < 0) {
547 dev_err(dev, "no more vmid to allocate\n");
548 return -ENOSPC;
549 }
550
551 pr_debug("vmid allocated: %d\n", allocated_vmid);
552
553 dqm->vmid_pasid[allocated_vmid] = pdd->pasid;
554
555 set_pasid_vmid_mapping(dqm, pdd->pasid, allocated_vmid);
556
557 qpd->vmid = allocated_vmid;
558 q->properties.vmid = allocated_vmid;
559
560 program_sh_mem_settings(dqm, qpd);
561
562 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled)
563 program_trap_handler_settings(dqm, qpd);
564
565 /* qpd->page_table_base is set earlier when register_process()
566 * is called, i.e. when the first queue is created.
567 */
568 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev,
569 qpd->vmid,
570 qpd->page_table_base);
571 /* invalidate the VM context after pasid and vmid mapping is set up */
572 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
573
574 if (dqm->dev->kfd2kgd->set_scratch_backing_va)
575 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
576 qpd->sh_hidden_private_base, qpd->vmid);
577
578 return 0;
579 }
580
flush_texture_cache_nocpsch(struct kfd_node * kdev,struct qcm_process_device * qpd)581 static int flush_texture_cache_nocpsch(struct kfd_node *kdev,
582 struct qcm_process_device *qpd)
583 {
584 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf;
585 int ret;
586
587 if (!qpd->ib_kaddr)
588 return -ENOMEM;
589
590 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
591 if (ret)
592 return ret;
593
594 return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid,
595 qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
596 pmf->release_mem_size / sizeof(uint32_t));
597 }
598
deallocate_vmid(struct device_queue_manager * dqm,struct qcm_process_device * qpd,struct queue * q)599 static void deallocate_vmid(struct device_queue_manager *dqm,
600 struct qcm_process_device *qpd,
601 struct queue *q)
602 {
603 struct device *dev = dqm->dev->adev->dev;
604
605 /* On GFX v7, CP doesn't flush TC at dequeue */
606 if (q->device->adev->asic_type == CHIP_HAWAII)
607 if (flush_texture_cache_nocpsch(q->device, qpd))
608 dev_err(dev, "Failed to flush TC\n");
609
610 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
611
612 /* Release the vmid mapping */
613 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
614 dqm->vmid_pasid[qpd->vmid] = 0;
615
616 qpd->vmid = 0;
617 q->properties.vmid = 0;
618 }
619
create_queue_nocpsch(struct device_queue_manager * dqm,struct queue * q,struct qcm_process_device * qpd,const struct kfd_criu_queue_priv_data * qd,const void * restore_mqd,const void * restore_ctl_stack)620 static int create_queue_nocpsch(struct device_queue_manager *dqm,
621 struct queue *q,
622 struct qcm_process_device *qpd,
623 const struct kfd_criu_queue_priv_data *qd,
624 const void *restore_mqd, const void *restore_ctl_stack)
625 {
626 struct mqd_manager *mqd_mgr;
627 int retval;
628
629 dqm_lock(dqm);
630
631 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
632 pr_warn("Can't create new usermode queue because %d queues were already created\n",
633 dqm->total_queue_count);
634 retval = -EPERM;
635 goto out_unlock;
636 }
637
638 if (list_empty(&qpd->queues_list)) {
639 retval = allocate_vmid(dqm, qpd, q);
640 if (retval)
641 goto out_unlock;
642 }
643 q->properties.vmid = qpd->vmid;
644 /*
645 * Eviction state logic: mark all queues as evicted, even ones
646 * not currently active. Restoring inactive queues later only
647 * updates the is_evicted flag but is a no-op otherwise.
648 */
649 q->properties.is_evicted = !!qpd->evicted;
650
651 q->properties.tba_addr = qpd->tba_addr;
652 q->properties.tma_addr = qpd->tma_addr;
653
654 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
655 q->properties.type)];
656 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
657 retval = allocate_hqd(dqm, q);
658 if (retval)
659 goto deallocate_vmid;
660 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
661 q->pipe, q->queue);
662 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
663 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
664 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);
665 if (retval)
666 goto deallocate_vmid;
667 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
668 }
669
670 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);
671 if (retval)
672 goto out_deallocate_hqd;
673
674 /* Temporarily release dqm lock to avoid a circular lock dependency */
675 dqm_unlock(dqm);
676 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
677 dqm_lock(dqm);
678
679 if (!q->mqd_mem_obj) {
680 retval = -ENOMEM;
681 goto out_deallocate_doorbell;
682 }
683
684 if (qd)
685 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
686 &q->properties, restore_mqd, restore_ctl_stack,
687 qd->ctl_stack_size);
688 else
689 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
690 &q->gart_mqd_addr, &q->properties);
691
692 if (q->properties.is_active) {
693 if (!dqm->sched_running) {
694 WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
695 goto add_queue_to_list;
696 }
697
698 if (WARN(q->process->mm != current->mm,
699 "should only run in user thread"))
700 retval = -EFAULT;
701 else
702 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
703 q->queue, &q->properties, current->mm);
704 if (retval)
705 goto out_free_mqd;
706 }
707
708 add_queue_to_list:
709 list_add(&q->list, &qpd->queues_list);
710 qpd->queue_count++;
711 if (q->properties.is_active)
712 increment_queue_count(dqm, qpd, q);
713
714 /*
715 * Unconditionally increment this counter, regardless of the queue's
716 * type or whether the queue is active.
717 */
718 dqm->total_queue_count++;
719 pr_debug("Total of %d queues are accountable so far\n",
720 dqm->total_queue_count);
721 goto out_unlock;
722
723 out_free_mqd:
724 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
725 out_deallocate_doorbell:
726 deallocate_doorbell(qpd, q);
727 out_deallocate_hqd:
728 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
729 deallocate_hqd(dqm, q);
730 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
731 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
732 deallocate_sdma_queue(dqm, q);
733 deallocate_vmid:
734 if (list_empty(&qpd->queues_list))
735 deallocate_vmid(dqm, qpd, q);
736 out_unlock:
737 dqm_unlock(dqm);
738 return retval;
739 }
740
allocate_hqd(struct device_queue_manager * dqm,struct queue * q)741 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
742 {
743 bool set;
744 int pipe, bit, i;
745
746 set = false;
747
748 for (pipe = dqm->next_pipe_to_allocate, i = 0;
749 i < get_pipes_per_mec(dqm);
750 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
751
752 if (!is_pipe_enabled(dqm, 0, pipe))
753 continue;
754
755 if (dqm->allocated_queues[pipe] != 0) {
756 bit = ffs(dqm->allocated_queues[pipe]) - 1;
757 dqm->allocated_queues[pipe] &= ~(1 << bit);
758 q->pipe = pipe;
759 q->queue = bit;
760 set = true;
761 break;
762 }
763 }
764
765 if (!set)
766 return -EBUSY;
767
768 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
769 /* horizontal hqd allocation */
770 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
771
772 return 0;
773 }
774
deallocate_hqd(struct device_queue_manager * dqm,struct queue * q)775 static inline void deallocate_hqd(struct device_queue_manager *dqm,
776 struct queue *q)
777 {
778 dqm->allocated_queues[q->pipe] |= (1 << q->queue);
779 }
780
781 #define SQ_IND_CMD_CMD_KILL 0x00000003
782 #define SQ_IND_CMD_MODE_BROADCAST 0x00000001
783
dbgdev_wave_reset_wavefronts(struct kfd_node * dev,struct kfd_process * p)784 static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p)
785 {
786 int status = 0;
787 unsigned int vmid;
788 uint16_t queried_pasid;
789 union SQ_CMD_BITS reg_sq_cmd;
790 union GRBM_GFX_INDEX_BITS reg_gfx_index;
791 struct kfd_process_device *pdd;
792 int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
793 int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
794 uint32_t xcc_mask = dev->xcc_mask;
795 int xcc_id;
796
797 reg_sq_cmd.u32All = 0;
798 reg_gfx_index.u32All = 0;
799
800 pr_debug("Killing all process wavefronts\n");
801
802 if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) {
803 dev_err(dev->adev->dev, "no vmid pasid mapping supported\n");
804 return -EOPNOTSUPP;
805 }
806
807 /* taking the VMID for that process on the safe way using PDD */
808 pdd = kfd_get_process_device_data(dev, p);
809 if (!pdd)
810 return -EFAULT;
811
812 /* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
813 * ATC_VMID15_PASID_MAPPING
814 * to check which VMID the current process is mapped to.
815 */
816
817 for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
818 status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info
819 (dev->adev, vmid, &queried_pasid);
820
821 if (status && queried_pasid == pdd->pasid) {
822 pr_debug("Killing wave fronts of vmid %d and process pid %d\n",
823 vmid, p->lead_thread->pid);
824 break;
825 }
826 }
827
828 if (vmid > last_vmid_to_scan) {
829 dev_err(dev->adev->dev, "Didn't find vmid for process pid %d\n",
830 p->lead_thread->pid);
831 return -EFAULT;
832 }
833
834 reg_gfx_index.bits.sh_broadcast_writes = 1;
835 reg_gfx_index.bits.se_broadcast_writes = 1;
836 reg_gfx_index.bits.instance_broadcast_writes = 1;
837 reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
838 reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL;
839 reg_sq_cmd.bits.vm_id = vmid;
840
841 for_each_inst(xcc_id, xcc_mask)
842 dev->kfd2kgd->wave_control_execute(
843 dev->adev, reg_gfx_index.u32All,
844 reg_sq_cmd.u32All, xcc_id);
845
846 return 0;
847 }
848
849 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
850 * to avoid asynchronized access
851 */
destroy_queue_nocpsch_locked(struct device_queue_manager * dqm,struct qcm_process_device * qpd,struct queue * q)852 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
853 struct qcm_process_device *qpd,
854 struct queue *q)
855 {
856 int retval;
857 struct mqd_manager *mqd_mgr;
858
859 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
860 q->properties.type)];
861
862 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
863 deallocate_hqd(dqm, q);
864 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
865 deallocate_sdma_queue(dqm, q);
866 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
867 deallocate_sdma_queue(dqm, q);
868 else {
869 pr_debug("q->properties.type %d is invalid\n",
870 q->properties.type);
871 return -EINVAL;
872 }
873 dqm->total_queue_count--;
874
875 deallocate_doorbell(qpd, q);
876
877 if (!dqm->sched_running) {
878 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
879 return 0;
880 }
881
882 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
883 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
884 KFD_UNMAP_LATENCY_MS,
885 q->pipe, q->queue);
886 if (retval == -ETIME)
887 qpd->reset_wavefronts = true;
888
889 list_del(&q->list);
890 if (list_empty(&qpd->queues_list)) {
891 if (qpd->reset_wavefronts) {
892 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
893 dqm->dev);
894 /* dbgdev_wave_reset_wavefronts has to be called before
895 * deallocate_vmid(), i.e. when vmid is still in use.
896 */
897 dbgdev_wave_reset_wavefronts(dqm->dev,
898 qpd->pqm->process);
899 qpd->reset_wavefronts = false;
900 }
901
902 deallocate_vmid(dqm, qpd, q);
903 }
904 qpd->queue_count--;
905 if (q->properties.is_active)
906 decrement_queue_count(dqm, qpd, q);
907
908 return retval;
909 }
910
destroy_queue_nocpsch(struct device_queue_manager * dqm,struct qcm_process_device * qpd,struct queue * q)911 static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
912 struct qcm_process_device *qpd,
913 struct queue *q)
914 {
915 int retval;
916 uint64_t sdma_val = 0;
917 struct device *dev = dqm->dev->adev->dev;
918 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
919 struct mqd_manager *mqd_mgr =
920 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];
921
922 /* Get the SDMA queue stats */
923 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
924 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
925 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
926 &sdma_val);
927 if (retval)
928 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
929 q->properties.queue_id);
930 }
931
932 dqm_lock(dqm);
933 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
934 if (!retval)
935 pdd->sdma_past_activity_counter += sdma_val;
936 dqm_unlock(dqm);
937
938 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
939
940 return retval;
941 }
942
update_queue(struct device_queue_manager * dqm,struct queue * q,struct mqd_update_info * minfo)943 static int update_queue(struct device_queue_manager *dqm, struct queue *q,
944 struct mqd_update_info *minfo)
945 {
946 int retval = 0;
947 struct device *dev = dqm->dev->adev->dev;
948 struct mqd_manager *mqd_mgr;
949 struct kfd_process_device *pdd;
950 bool prev_active = false;
951
952 dqm_lock(dqm);
953 pdd = kfd_get_process_device_data(q->device, q->process);
954 if (!pdd) {
955 retval = -ENODEV;
956 goto out_unlock;
957 }
958 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
959 q->properties.type)];
960
961 /* Save previous activity state for counters */
962 prev_active = q->properties.is_active;
963
964 /* Make sure the queue is unmapped before updating the MQD */
965 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
966 if (!dqm->dev->kfd->shared_resources.enable_mes)
967 retval = unmap_queues_cpsch(dqm,
968 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false);
969 else if (prev_active)
970 retval = remove_queue_mes(dqm, q, &pdd->qpd);
971
972 /* queue is reset so inaccessable */
973 if (pdd->has_reset_queue) {
974 retval = -EACCES;
975 goto out_unlock;
976 }
977
978 if (retval) {
979 dev_err(dev, "unmap queue failed\n");
980 goto out_unlock;
981 }
982 } else if (prev_active &&
983 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
984 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
985 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
986
987 if (!dqm->sched_running) {
988 WARN_ONCE(1, "Update non-HWS queue while stopped\n");
989 goto out_unlock;
990 }
991
992 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
993 (dqm->dev->kfd->cwsr_enabled ?
994 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
995 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
996 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
997 if (retval) {
998 dev_err(dev, "destroy mqd failed\n");
999 goto out_unlock;
1000 }
1001 }
1002
1003 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo);
1004
1005 /*
1006 * check active state vs. the previous state and modify
1007 * counter accordingly. map_queues_cpsch uses the
1008 * dqm->active_queue_count to determine whether a new runlist must be
1009 * uploaded.
1010 */
1011 if (q->properties.is_active && !prev_active) {
1012 increment_queue_count(dqm, &pdd->qpd, q);
1013 } else if (!q->properties.is_active && prev_active) {
1014 decrement_queue_count(dqm, &pdd->qpd, q);
1015 } else if (q->gws && !q->properties.is_gws) {
1016 if (q->properties.is_active) {
1017 dqm->gws_queue_count++;
1018 pdd->qpd.mapped_gws_queue = true;
1019 }
1020 q->properties.is_gws = true;
1021 } else if (!q->gws && q->properties.is_gws) {
1022 if (q->properties.is_active) {
1023 dqm->gws_queue_count--;
1024 pdd->qpd.mapped_gws_queue = false;
1025 }
1026 q->properties.is_gws = false;
1027 }
1028
1029 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
1030 if (!dqm->dev->kfd->shared_resources.enable_mes)
1031 retval = map_queues_cpsch(dqm);
1032 else if (q->properties.is_active)
1033 retval = add_queue_mes(dqm, q, &pdd->qpd);
1034 } else if (q->properties.is_active &&
1035 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1036 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1037 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1038 if (WARN(q->process->mm != current->mm,
1039 "should only run in user thread"))
1040 retval = -EFAULT;
1041 else
1042 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
1043 q->pipe, q->queue,
1044 &q->properties, current->mm);
1045 }
1046
1047 out_unlock:
1048 dqm_unlock(dqm);
1049 return retval;
1050 }
1051
1052 /* suspend_single_queue does not lock the dqm like the
1053 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should
1054 * lock the dqm before calling, and unlock after calling.
1055 *
1056 * The reason we don't lock the dqm is because this function may be
1057 * called on multiple queues in a loop, so rather than locking/unlocking
1058 * multiple times, we will just keep the dqm locked for all of the calls.
1059 */
suspend_single_queue(struct device_queue_manager * dqm,struct kfd_process_device * pdd,struct queue * q)1060 static int suspend_single_queue(struct device_queue_manager *dqm,
1061 struct kfd_process_device *pdd,
1062 struct queue *q)
1063 {
1064 bool is_new;
1065
1066 if (q->properties.is_suspended)
1067 return 0;
1068
1069 pr_debug("Suspending process pid %d queue [%i]\n",
1070 pdd->process->lead_thread->pid,
1071 q->properties.queue_id);
1072
1073 is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW);
1074
1075 if (is_new || q->properties.is_being_destroyed) {
1076 pr_debug("Suspend: skip %s queue id %i\n",
1077 is_new ? "new" : "destroyed",
1078 q->properties.queue_id);
1079 return -EBUSY;
1080 }
1081
1082 q->properties.is_suspended = true;
1083 if (q->properties.is_active) {
1084 if (dqm->dev->kfd->shared_resources.enable_mes) {
1085 int r = remove_queue_mes(dqm, q, &pdd->qpd);
1086
1087 if (r)
1088 return r;
1089 }
1090
1091 decrement_queue_count(dqm, &pdd->qpd, q);
1092 q->properties.is_active = false;
1093 }
1094
1095 return 0;
1096 }
1097
1098 /* resume_single_queue does not lock the dqm like the functions
1099 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should
1100 * lock the dqm before calling, and unlock after calling.
1101 *
1102 * The reason we don't lock the dqm is because this function may be
1103 * called on multiple queues in a loop, so rather than locking/unlocking
1104 * multiple times, we will just keep the dqm locked for all of the calls.
1105 */
resume_single_queue(struct device_queue_manager * dqm,struct qcm_process_device * qpd,struct queue * q)1106 static int resume_single_queue(struct device_queue_manager *dqm,
1107 struct qcm_process_device *qpd,
1108 struct queue *q)
1109 {
1110 struct kfd_process_device *pdd;
1111
1112 if (!q->properties.is_suspended)
1113 return 0;
1114
1115 pdd = qpd_to_pdd(qpd);
1116
1117 pr_debug("Restoring from suspend process pid %d queue [%i]\n",
1118 pdd->process->lead_thread->pid,
1119 q->properties.queue_id);
1120
1121 q->properties.is_suspended = false;
1122
1123 if (QUEUE_IS_ACTIVE(q->properties)) {
1124 if (dqm->dev->kfd->shared_resources.enable_mes) {
1125 int r = add_queue_mes(dqm, q, &pdd->qpd);
1126
1127 if (r)
1128 return r;
1129 }
1130
1131 q->properties.is_active = true;
1132 increment_queue_count(dqm, qpd, q);
1133 }
1134
1135 return 0;
1136 }
1137
evict_process_queues_nocpsch(struct device_queue_manager * dqm,struct qcm_process_device * qpd)1138 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
1139 struct qcm_process_device *qpd)
1140 {
1141 struct queue *q;
1142 struct mqd_manager *mqd_mgr;
1143 struct kfd_process_device *pdd;
1144 int retval, ret = 0;
1145
1146 dqm_lock(dqm);
1147 if (qpd->evicted++ > 0) /* already evicted, do nothing */
1148 goto out;
1149
1150 pdd = qpd_to_pdd(qpd);
1151 pr_debug_ratelimited("Evicting process pid %d queues\n",
1152 pdd->process->lead_thread->pid);
1153
1154 pdd->last_evict_timestamp = get_jiffies_64();
1155 /* Mark all queues as evicted. Deactivate all active queues on
1156 * the qpd.
1157 */
1158 list_for_each_entry(q, &qpd->queues_list, list) {
1159 q->properties.is_evicted = true;
1160 if (!q->properties.is_active)
1161 continue;
1162
1163 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1164 q->properties.type)];
1165 q->properties.is_active = false;
1166 decrement_queue_count(dqm, qpd, q);
1167
1168 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
1169 continue;
1170
1171 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
1172 (dqm->dev->kfd->cwsr_enabled ?
1173 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
1174 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
1175 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
1176 if (retval && !ret)
1177 /* Return the first error, but keep going to
1178 * maintain a consistent eviction state
1179 */
1180 ret = retval;
1181 }
1182
1183 out:
1184 dqm_unlock(dqm);
1185 return ret;
1186 }
1187
evict_process_queues_cpsch(struct device_queue_manager * dqm,struct qcm_process_device * qpd)1188 static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
1189 struct qcm_process_device *qpd)
1190 {
1191 struct queue *q;
1192 struct device *dev = dqm->dev->adev->dev;
1193 struct kfd_process_device *pdd;
1194 int retval = 0;
1195
1196 dqm_lock(dqm);
1197 if (qpd->evicted++ > 0) /* already evicted, do nothing */
1198 goto out;
1199
1200 pdd = qpd_to_pdd(qpd);
1201
1202 /* The debugger creates processes that temporarily have not acquired
1203 * all VMs for all devices and has no VMs itself.
1204 * Skip queue eviction on process eviction.
1205 */
1206 if (!pdd->drm_priv)
1207 goto out;
1208
1209 pr_debug_ratelimited("Evicting process pid %d queues\n",
1210 pdd->process->lead_thread->pid);
1211
1212 /* Mark all queues as evicted. Deactivate all active queues on
1213 * the qpd.
1214 */
1215 list_for_each_entry(q, &qpd->queues_list, list) {
1216 q->properties.is_evicted = true;
1217 if (!q->properties.is_active)
1218 continue;
1219
1220 q->properties.is_active = false;
1221 decrement_queue_count(dqm, qpd, q);
1222
1223 if (dqm->dev->kfd->shared_resources.enable_mes) {
1224 int err;
1225
1226 err = remove_queue_mes(dqm, q, qpd);
1227 if (err) {
1228 dev_err(dev, "Failed to evict queue %d\n",
1229 q->properties.queue_id);
1230 retval = err;
1231 }
1232 }
1233 }
1234 pdd->last_evict_timestamp = get_jiffies_64();
1235 if (!dqm->dev->kfd->shared_resources.enable_mes)
1236 retval = execute_queues_cpsch(dqm,
1237 qpd->is_debug ?
1238 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
1239 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
1240 USE_DEFAULT_GRACE_PERIOD);
1241
1242 out:
1243 dqm_unlock(dqm);
1244 return retval;
1245 }
1246
restore_process_queues_nocpsch(struct device_queue_manager * dqm,struct qcm_process_device * qpd)1247 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
1248 struct qcm_process_device *qpd)
1249 {
1250 struct mm_struct *mm = NULL;
1251 struct queue *q;
1252 struct mqd_manager *mqd_mgr;
1253 struct kfd_process_device *pdd;
1254 uint64_t pd_base;
1255 uint64_t eviction_duration;
1256 int retval, ret = 0;
1257
1258 pdd = qpd_to_pdd(qpd);
1259 /* Retrieve PD base */
1260 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1261
1262 dqm_lock(dqm);
1263 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1264 goto out;
1265 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1266 qpd->evicted--;
1267 goto out;
1268 }
1269
1270 pr_debug_ratelimited("Restoring process pid %d queues\n",
1271 pdd->process->lead_thread->pid);
1272
1273 /* Update PD Base in QPD */
1274 qpd->page_table_base = pd_base;
1275 pr_debug("Updated PD address to 0x%llx\n", pd_base);
1276
1277 if (!list_empty(&qpd->queues_list)) {
1278 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
1279 dqm->dev->adev,
1280 qpd->vmid,
1281 qpd->page_table_base);
1282 kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
1283 }
1284
1285 /* Take a safe reference to the mm_struct, which may otherwise
1286 * disappear even while the kfd_process is still referenced.
1287 */
1288 mm = get_task_mm(pdd->process->lead_thread);
1289 if (!mm) {
1290 ret = -EFAULT;
1291 goto out;
1292 }
1293
1294 /* Remove the eviction flags. Activate queues that are not
1295 * inactive for other reasons.
1296 */
1297 list_for_each_entry(q, &qpd->queues_list, list) {
1298 q->properties.is_evicted = false;
1299 if (!QUEUE_IS_ACTIVE(q->properties))
1300 continue;
1301
1302 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1303 q->properties.type)];
1304 q->properties.is_active = true;
1305 increment_queue_count(dqm, qpd, q);
1306
1307 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
1308 continue;
1309
1310 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
1311 q->queue, &q->properties, mm);
1312 if (retval && !ret)
1313 /* Return the first error, but keep going to
1314 * maintain a consistent eviction state
1315 */
1316 ret = retval;
1317 }
1318 qpd->evicted = 0;
1319 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1320 atomic64_add(eviction_duration, &pdd->evict_duration_counter);
1321 out:
1322 if (mm)
1323 mmput(mm);
1324 dqm_unlock(dqm);
1325 return ret;
1326 }
1327
restore_process_queues_cpsch(struct device_queue_manager * dqm,struct qcm_process_device * qpd)1328 static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
1329 struct qcm_process_device *qpd)
1330 {
1331 struct queue *q;
1332 struct device *dev = dqm->dev->adev->dev;
1333 struct kfd_process_device *pdd;
1334 uint64_t eviction_duration;
1335 int retval = 0;
1336
1337 pdd = qpd_to_pdd(qpd);
1338
1339 dqm_lock(dqm);
1340 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1341 goto out;
1342 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1343 qpd->evicted--;
1344 goto out;
1345 }
1346
1347 /* The debugger creates processes that temporarily have not acquired
1348 * all VMs for all devices and has no VMs itself.
1349 * Skip queue restore on process restore.
1350 */
1351 if (!pdd->drm_priv)
1352 goto vm_not_acquired;
1353
1354 pr_debug_ratelimited("Restoring process pid %d queues\n",
1355 pdd->process->lead_thread->pid);
1356
1357 /* Update PD Base in QPD */
1358 qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1359 pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base);
1360
1361 /* activate all active queues on the qpd */
1362 list_for_each_entry(q, &qpd->queues_list, list) {
1363 q->properties.is_evicted = false;
1364 if (!QUEUE_IS_ACTIVE(q->properties))
1365 continue;
1366
1367 q->properties.is_active = true;
1368 increment_queue_count(dqm, &pdd->qpd, q);
1369
1370 if (dqm->dev->kfd->shared_resources.enable_mes) {
1371 retval = add_queue_mes(dqm, q, qpd);
1372 if (retval) {
1373 dev_err(dev, "Failed to restore queue %d\n",
1374 q->properties.queue_id);
1375 goto out;
1376 }
1377 }
1378 }
1379 if (!dqm->dev->kfd->shared_resources.enable_mes)
1380 retval = execute_queues_cpsch(dqm,
1381 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
1382 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1383 atomic64_add(eviction_duration, &pdd->evict_duration_counter);
1384 vm_not_acquired:
1385 qpd->evicted = 0;
1386 out:
1387 dqm_unlock(dqm);
1388 return retval;
1389 }
1390
register_process(struct device_queue_manager * dqm,struct qcm_process_device * qpd)1391 static int register_process(struct device_queue_manager *dqm,
1392 struct qcm_process_device *qpd)
1393 {
1394 struct device_process_node *n;
1395 struct kfd_process_device *pdd;
1396 uint64_t pd_base;
1397 int retval;
1398
1399 n = kzalloc(sizeof(*n), GFP_KERNEL);
1400 if (!n)
1401 return -ENOMEM;
1402
1403 n->qpd = qpd;
1404
1405 pdd = qpd_to_pdd(qpd);
1406 /* Retrieve PD base */
1407 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1408
1409 dqm_lock(dqm);
1410 list_add(&n->list, &dqm->queues);
1411
1412 /* Update PD Base in QPD */
1413 qpd->page_table_base = pd_base;
1414 pr_debug("Updated PD address to 0x%llx\n", pd_base);
1415
1416 retval = dqm->asic_ops.update_qpd(dqm, qpd);
1417
1418 dqm->processes_count++;
1419
1420 dqm_unlock(dqm);
1421
1422 /* Outside the DQM lock because under the DQM lock we can't do
1423 * reclaim or take other locks that others hold while reclaiming.
1424 */
1425 kfd_inc_compute_active(dqm->dev);
1426
1427 return retval;
1428 }
1429
unregister_process(struct device_queue_manager * dqm,struct qcm_process_device * qpd)1430 static int unregister_process(struct device_queue_manager *dqm,
1431 struct qcm_process_device *qpd)
1432 {
1433 int retval;
1434 struct device_process_node *cur, *next;
1435
1436 pr_debug("qpd->queues_list is %s\n",
1437 list_empty(&qpd->queues_list) ? "empty" : "not empty");
1438
1439 retval = 0;
1440 dqm_lock(dqm);
1441
1442 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
1443 if (qpd == cur->qpd) {
1444 list_del(&cur->list);
1445 kfree(cur);
1446 dqm->processes_count--;
1447 goto out;
1448 }
1449 }
1450 /* qpd not found in dqm list */
1451 retval = 1;
1452 out:
1453 dqm_unlock(dqm);
1454
1455 /* Outside the DQM lock because under the DQM lock we can't do
1456 * reclaim or take other locks that others hold while reclaiming.
1457 */
1458 if (!retval)
1459 kfd_dec_compute_active(dqm->dev);
1460
1461 return retval;
1462 }
1463
1464 static int
set_pasid_vmid_mapping(struct device_queue_manager * dqm,u32 pasid,unsigned int vmid)1465 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
1466 unsigned int vmid)
1467 {
1468 uint32_t xcc_mask = dqm->dev->xcc_mask;
1469 int xcc_id, ret;
1470
1471 for_each_inst(xcc_id, xcc_mask) {
1472 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
1473 dqm->dev->adev, pasid, vmid, xcc_id);
1474 if (ret)
1475 break;
1476 }
1477
1478 return ret;
1479 }
1480
init_interrupts(struct device_queue_manager * dqm)1481 static void init_interrupts(struct device_queue_manager *dqm)
1482 {
1483 uint32_t xcc_mask = dqm->dev->xcc_mask;
1484 unsigned int i, xcc_id;
1485
1486 for_each_inst(xcc_id, xcc_mask) {
1487 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) {
1488 if (is_pipe_enabled(dqm, 0, i)) {
1489 dqm->dev->kfd2kgd->init_interrupts(
1490 dqm->dev->adev, i, xcc_id);
1491 }
1492 }
1493 }
1494 }
1495
initialize_nocpsch(struct device_queue_manager * dqm)1496 static int initialize_nocpsch(struct device_queue_manager *dqm)
1497 {
1498 int pipe, queue;
1499
1500 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1501
1502 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
1503 sizeof(unsigned int), GFP_KERNEL);
1504 if (!dqm->allocated_queues)
1505 return -ENOMEM;
1506
1507 mutex_init(&dqm->lock_hidden);
1508 INIT_LIST_HEAD(&dqm->queues);
1509 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;
1510 dqm->active_cp_queue_count = 0;
1511 dqm->gws_queue_count = 0;
1512
1513 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1514 int pipe_offset = pipe * get_queues_per_pipe(dqm);
1515
1516 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
1517 if (test_bit(pipe_offset + queue,
1518 dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1519 dqm->allocated_queues[pipe] |= 1 << queue;
1520 }
1521
1522 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
1523
1524 init_sdma_bitmaps(dqm);
1525
1526 return 0;
1527 }
1528
uninitialize(struct device_queue_manager * dqm)1529 static void uninitialize(struct device_queue_manager *dqm)
1530 {
1531 int i;
1532
1533 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);
1534
1535 kfree(dqm->allocated_queues);
1536 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
1537 kfree(dqm->mqd_mgrs[i]);
1538 mutex_destroy(&dqm->lock_hidden);
1539 }
1540
start_nocpsch(struct device_queue_manager * dqm)1541 static int start_nocpsch(struct device_queue_manager *dqm)
1542 {
1543 int r = 0;
1544
1545 pr_info("SW scheduler is used");
1546 init_interrupts(dqm);
1547
1548 if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1549 r = pm_init(&dqm->packet_mgr, dqm);
1550 if (!r)
1551 dqm->sched_running = true;
1552
1553 return r;
1554 }
1555
stop_nocpsch(struct device_queue_manager * dqm)1556 static int stop_nocpsch(struct device_queue_manager *dqm)
1557 {
1558 dqm_lock(dqm);
1559 if (!dqm->sched_running) {
1560 dqm_unlock(dqm);
1561 return 0;
1562 }
1563
1564 if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1565 pm_uninit(&dqm->packet_mgr);
1566 dqm->sched_running = false;
1567 dqm_unlock(dqm);
1568
1569 return 0;
1570 }
1571
allocate_sdma_queue(struct device_queue_manager * dqm,struct queue * q,const uint32_t * restore_sdma_id)1572 static int allocate_sdma_queue(struct device_queue_manager *dqm,
1573 struct queue *q, const uint32_t *restore_sdma_id)
1574 {
1575 struct device *dev = dqm->dev->adev->dev;
1576 int bit;
1577
1578 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1579 if (bitmap_empty(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES)) {
1580 dev_err(dev, "No more SDMA queue to allocate\n");
1581 return -ENOMEM;
1582 }
1583
1584 if (restore_sdma_id) {
1585 /* Re-use existing sdma_id */
1586 if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {
1587 dev_err(dev, "SDMA queue already in use\n");
1588 return -EBUSY;
1589 }
1590 clear_bit(*restore_sdma_id, dqm->sdma_bitmap);
1591 q->sdma_id = *restore_sdma_id;
1592 } else {
1593 /* Find first available sdma_id */
1594 bit = find_first_bit(dqm->sdma_bitmap,
1595 get_num_sdma_queues(dqm));
1596 clear_bit(bit, dqm->sdma_bitmap);
1597 q->sdma_id = bit;
1598 }
1599
1600 q->properties.sdma_engine_id =
1601 q->sdma_id % kfd_get_num_sdma_engines(dqm->dev);
1602 q->properties.sdma_queue_id = q->sdma_id /
1603 kfd_get_num_sdma_engines(dqm->dev);
1604 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1605 if (bitmap_empty(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES)) {
1606 dev_err(dev, "No more XGMI SDMA queue to allocate\n");
1607 return -ENOMEM;
1608 }
1609 if (restore_sdma_id) {
1610 /* Re-use existing sdma_id */
1611 if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {
1612 dev_err(dev, "SDMA queue already in use\n");
1613 return -EBUSY;
1614 }
1615 clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap);
1616 q->sdma_id = *restore_sdma_id;
1617 } else {
1618 bit = find_first_bit(dqm->xgmi_sdma_bitmap,
1619 get_num_xgmi_sdma_queues(dqm));
1620 clear_bit(bit, dqm->xgmi_sdma_bitmap);
1621 q->sdma_id = bit;
1622 }
1623 /* sdma_engine_id is sdma id including
1624 * both PCIe-optimized SDMAs and XGMI-
1625 * optimized SDMAs. The calculation below
1626 * assumes the first N engines are always
1627 * PCIe-optimized ones
1628 */
1629 q->properties.sdma_engine_id =
1630 kfd_get_num_sdma_engines(dqm->dev) +
1631 q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev);
1632 q->properties.sdma_queue_id = q->sdma_id /
1633 kfd_get_num_xgmi_sdma_engines(dqm->dev);
1634 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
1635 int i, num_queues, num_engines, eng_offset = 0, start_engine;
1636 bool free_bit_found = false, is_xgmi = false;
1637
1638 if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) {
1639 num_queues = get_num_sdma_queues(dqm);
1640 num_engines = kfd_get_num_sdma_engines(dqm->dev);
1641 q->properties.type = KFD_QUEUE_TYPE_SDMA;
1642 } else {
1643 num_queues = get_num_xgmi_sdma_queues(dqm);
1644 num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev);
1645 eng_offset = kfd_get_num_sdma_engines(dqm->dev);
1646 q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI;
1647 is_xgmi = true;
1648 }
1649
1650 /* Scan available bit based on target engine ID. */
1651 start_engine = q->properties.sdma_engine_id - eng_offset;
1652 for (i = start_engine; i < num_queues; i += num_engines) {
1653
1654 if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap))
1655 continue;
1656
1657 clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap);
1658 q->sdma_id = i;
1659 q->properties.sdma_queue_id = q->sdma_id / num_engines;
1660 free_bit_found = true;
1661 break;
1662 }
1663
1664 if (!free_bit_found) {
1665 dev_err(dev, "No more SDMA queue to allocate for target ID %i\n",
1666 q->properties.sdma_engine_id);
1667 return -ENOMEM;
1668 }
1669 }
1670
1671 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
1672 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
1673
1674 return 0;
1675 }
1676
deallocate_sdma_queue(struct device_queue_manager * dqm,struct queue * q)1677 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1678 struct queue *q)
1679 {
1680 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1681 if (q->sdma_id >= get_num_sdma_queues(dqm))
1682 return;
1683 set_bit(q->sdma_id, dqm->sdma_bitmap);
1684 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1685 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
1686 return;
1687 set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap);
1688 }
1689 }
1690
1691 /*
1692 * Device Queue Manager implementation for cp scheduler
1693 */
1694
set_sched_resources(struct device_queue_manager * dqm)1695 static int set_sched_resources(struct device_queue_manager *dqm)
1696 {
1697 int i, mec;
1698 struct scheduling_resources res;
1699 struct device *dev = dqm->dev->adev->dev;
1700
1701 res.vmid_mask = dqm->dev->compute_vmid_bitmap;
1702
1703 res.queue_mask = 0;
1704 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
1705 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
1706 / dqm->dev->kfd->shared_resources.num_pipe_per_mec;
1707
1708 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1709 continue;
1710
1711 /* only acquire queues from the first MEC */
1712 if (mec > 0)
1713 continue;
1714
1715 /* This situation may be hit in the future if a new HW
1716 * generation exposes more than 64 queues. If so, the
1717 * definition of res.queue_mask needs updating
1718 */
1719 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
1720 dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i);
1721 break;
1722 }
1723
1724 res.queue_mask |= 1ull
1725 << amdgpu_queue_mask_bit_to_set_resource_bit(
1726 dqm->dev->adev, i);
1727 }
1728 res.gws_mask = ~0ull;
1729 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
1730
1731 pr_debug("Scheduling resources:\n"
1732 "vmid mask: 0x%8X\n"
1733 "queue mask: 0x%8llX\n",
1734 res.vmid_mask, res.queue_mask);
1735
1736 return pm_send_set_resources(&dqm->packet_mgr, &res);
1737 }
1738
initialize_cpsch(struct device_queue_manager * dqm)1739 static int initialize_cpsch(struct device_queue_manager *dqm)
1740 {
1741 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1742
1743 mutex_init(&dqm->lock_hidden);
1744 INIT_LIST_HEAD(&dqm->queues);
1745 dqm->active_queue_count = dqm->processes_count = 0;
1746 dqm->active_cp_queue_count = 0;
1747 dqm->gws_queue_count = 0;
1748 dqm->active_runlist = false;
1749 dqm->trap_debug_vmid = 0;
1750
1751 init_sdma_bitmaps(dqm);
1752
1753 update_dqm_wait_times(dqm);
1754 return 0;
1755 }
1756
1757 /* halt_cpsch:
1758 * Unmap queues so the schedule doesn't continue remaining jobs in the queue.
1759 * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch
1760 * is called.
1761 */
halt_cpsch(struct device_queue_manager * dqm)1762 static int halt_cpsch(struct device_queue_manager *dqm)
1763 {
1764 int ret = 0;
1765
1766 dqm_lock(dqm);
1767 if (!dqm->sched_running) {
1768 dqm_unlock(dqm);
1769 return 0;
1770 }
1771
1772 WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n");
1773
1774 if (!dqm->is_hws_hang) {
1775 if (!dqm->dev->kfd->shared_resources.enable_mes)
1776 ret = unmap_queues_cpsch(dqm,
1777 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
1778 USE_DEFAULT_GRACE_PERIOD, false);
1779 else
1780 ret = remove_all_kfd_queues_mes(dqm);
1781 }
1782 dqm->sched_halt = true;
1783 dqm_unlock(dqm);
1784
1785 return ret;
1786 }
1787
1788 /* unhalt_cpsch
1789 * Unset dqm->sched_halt and map queues back to runlist
1790 */
unhalt_cpsch(struct device_queue_manager * dqm)1791 static int unhalt_cpsch(struct device_queue_manager *dqm)
1792 {
1793 int ret = 0;
1794
1795 dqm_lock(dqm);
1796 if (!dqm->sched_running || !dqm->sched_halt) {
1797 WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n");
1798 dqm_unlock(dqm);
1799 return 0;
1800 }
1801 dqm->sched_halt = false;
1802 if (!dqm->dev->kfd->shared_resources.enable_mes)
1803 ret = execute_queues_cpsch(dqm,
1804 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
1805 0, USE_DEFAULT_GRACE_PERIOD);
1806 else
1807 ret = add_all_kfd_queues_mes(dqm);
1808
1809 dqm_unlock(dqm);
1810
1811 return ret;
1812 }
1813
start_cpsch(struct device_queue_manager * dqm)1814 static int start_cpsch(struct device_queue_manager *dqm)
1815 {
1816 struct device *dev = dqm->dev->adev->dev;
1817 int retval, num_hw_queue_slots;
1818
1819 retval = 0;
1820
1821 dqm_lock(dqm);
1822
1823 if (!dqm->dev->kfd->shared_resources.enable_mes) {
1824 retval = pm_init(&dqm->packet_mgr, dqm);
1825 if (retval)
1826 goto fail_packet_manager_init;
1827
1828 retval = set_sched_resources(dqm);
1829 if (retval)
1830 goto fail_set_sched_resources;
1831 }
1832 pr_debug("Allocating fence memory\n");
1833
1834 /* allocate fence memory on the gart */
1835 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
1836 &dqm->fence_mem);
1837
1838 if (retval)
1839 goto fail_allocate_vidmem;
1840
1841 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr;
1842 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
1843
1844 init_interrupts(dqm);
1845
1846 /* clear hang status when driver try to start the hw scheduler */
1847 dqm->sched_running = true;
1848
1849 if (!dqm->dev->kfd->shared_resources.enable_mes) {
1850 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr,
1851 KFD_DEQUEUE_WAIT_INIT, 0 /* unused */))
1852 dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
1853 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
1854 }
1855
1856 /* setup per-queue reset detection buffer */
1857 num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe *
1858 dqm->dev->kfd->shared_resources.num_pipe_per_mec *
1859 NUM_XCC(dqm->dev->xcc_mask);
1860
1861 dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info);
1862 dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL);
1863
1864 if (!dqm->detect_hang_info) {
1865 retval = -ENOMEM;
1866 goto fail_detect_hang_buffer;
1867 }
1868
1869 dqm_unlock(dqm);
1870
1871 return 0;
1872 fail_detect_hang_buffer:
1873 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
1874 fail_allocate_vidmem:
1875 fail_set_sched_resources:
1876 if (!dqm->dev->kfd->shared_resources.enable_mes)
1877 pm_uninit(&dqm->packet_mgr);
1878 fail_packet_manager_init:
1879 dqm_unlock(dqm);
1880 return retval;
1881 }
1882
stop_cpsch(struct device_queue_manager * dqm)1883 static int stop_cpsch(struct device_queue_manager *dqm)
1884 {
1885 dqm_lock(dqm);
1886 if (!dqm->sched_running) {
1887 dqm_unlock(dqm);
1888 return 0;
1889 }
1890
1891 if (!dqm->dev->kfd->shared_resources.enable_mes)
1892 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false);
1893 else
1894 remove_all_kfd_queues_mes(dqm);
1895
1896 dqm->sched_running = false;
1897
1898 if (!dqm->dev->kfd->shared_resources.enable_mes)
1899 pm_release_ib(&dqm->packet_mgr);
1900
1901 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
1902 if (!dqm->dev->kfd->shared_resources.enable_mes)
1903 pm_uninit(&dqm->packet_mgr);
1904 kfree(dqm->detect_hang_info);
1905 dqm->detect_hang_info = NULL;
1906 dqm_unlock(dqm);
1907
1908 return 0;
1909 }
1910
create_kernel_queue_cpsch(struct device_queue_manager * dqm,struct kernel_queue * kq,struct qcm_process_device * qpd)1911 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1912 struct kernel_queue *kq,
1913 struct qcm_process_device *qpd)
1914 {
1915 dqm_lock(dqm);
1916 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1917 pr_warn("Can't create new kernel queue because %d queues were already created\n",
1918 dqm->total_queue_count);
1919 dqm_unlock(dqm);
1920 return -EPERM;
1921 }
1922
1923 /*
1924 * Unconditionally increment this counter, regardless of the queue's
1925 * type or whether the queue is active.
1926 */
1927 dqm->total_queue_count++;
1928 pr_debug("Total of %d queues are accountable so far\n",
1929 dqm->total_queue_count);
1930
1931 list_add(&kq->list, &qpd->priv_queue_list);
1932 increment_queue_count(dqm, qpd, kq->queue);
1933 qpd->is_debug = true;
1934 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
1935 USE_DEFAULT_GRACE_PERIOD);
1936 dqm_unlock(dqm);
1937
1938 return 0;
1939 }
1940
destroy_kernel_queue_cpsch(struct device_queue_manager * dqm,struct kernel_queue * kq,struct qcm_process_device * qpd)1941 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1942 struct kernel_queue *kq,
1943 struct qcm_process_device *qpd)
1944 {
1945 dqm_lock(dqm);
1946 list_del(&kq->list);
1947 decrement_queue_count(dqm, qpd, kq->queue);
1948 qpd->is_debug = false;
1949 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
1950 USE_DEFAULT_GRACE_PERIOD);
1951 /*
1952 * Unconditionally decrement this counter, regardless of the queue's
1953 * type.
1954 */
1955 dqm->total_queue_count--;
1956 pr_debug("Total of %d queues are accountable so far\n",
1957 dqm->total_queue_count);
1958 dqm_unlock(dqm);
1959 }
1960
create_queue_cpsch(struct device_queue_manager * dqm,struct queue * q,struct qcm_process_device * qpd,const struct kfd_criu_queue_priv_data * qd,const void * restore_mqd,const void * restore_ctl_stack)1961 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
1962 struct qcm_process_device *qpd,
1963 const struct kfd_criu_queue_priv_data *qd,
1964 const void *restore_mqd, const void *restore_ctl_stack)
1965 {
1966 int retval;
1967 struct mqd_manager *mqd_mgr;
1968
1969 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1970 pr_warn("Can't create new usermode queue because %d queues were already created\n",
1971 dqm->total_queue_count);
1972 retval = -EPERM;
1973 goto out;
1974 }
1975
1976 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1977 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI ||
1978 q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
1979 dqm_lock(dqm);
1980 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);
1981 dqm_unlock(dqm);
1982 if (retval)
1983 goto out;
1984 }
1985
1986 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);
1987 if (retval)
1988 goto out_deallocate_sdma_queue;
1989
1990 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1991 q->properties.type)];
1992
1993 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1994 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1995 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
1996 q->properties.tba_addr = qpd->tba_addr;
1997 q->properties.tma_addr = qpd->tma_addr;
1998 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
1999 if (!q->mqd_mem_obj) {
2000 retval = -ENOMEM;
2001 goto out_deallocate_doorbell;
2002 }
2003
2004 dqm_lock(dqm);
2005 /*
2006 * Eviction state logic: mark all queues as evicted, even ones
2007 * not currently active. Restoring inactive queues later only
2008 * updates the is_evicted flag but is a no-op otherwise.
2009 */
2010 q->properties.is_evicted = !!qpd->evicted;
2011 q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled &&
2012 kfd_dbg_has_cwsr_workaround(q->device);
2013
2014 if (qd)
2015 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
2016 &q->properties, restore_mqd, restore_ctl_stack,
2017 qd->ctl_stack_size);
2018 else
2019 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
2020 &q->gart_mqd_addr, &q->properties);
2021
2022 list_add(&q->list, &qpd->queues_list);
2023 qpd->queue_count++;
2024
2025 if (q->properties.is_active) {
2026 increment_queue_count(dqm, qpd, q);
2027
2028 if (!dqm->dev->kfd->shared_resources.enable_mes)
2029 retval = execute_queues_cpsch(dqm,
2030 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
2031 else
2032 retval = add_queue_mes(dqm, q, qpd);
2033 if (retval)
2034 goto cleanup_queue;
2035 }
2036
2037 /*
2038 * Unconditionally increment this counter, regardless of the queue's
2039 * type or whether the queue is active.
2040 */
2041 dqm->total_queue_count++;
2042
2043 pr_debug("Total of %d queues are accountable so far\n",
2044 dqm->total_queue_count);
2045
2046 dqm_unlock(dqm);
2047 return retval;
2048
2049 cleanup_queue:
2050 qpd->queue_count--;
2051 list_del(&q->list);
2052 if (q->properties.is_active)
2053 decrement_queue_count(dqm, qpd, q);
2054 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2055 dqm_unlock(dqm);
2056 out_deallocate_doorbell:
2057 deallocate_doorbell(qpd, q);
2058 out_deallocate_sdma_queue:
2059 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2060 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
2061 dqm_lock(dqm);
2062 deallocate_sdma_queue(dqm, q);
2063 dqm_unlock(dqm);
2064 }
2065 out:
2066 return retval;
2067 }
2068
amdkfd_fence_wait_timeout(struct device_queue_manager * dqm,uint64_t fence_value,unsigned int timeout_ms)2069 int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm,
2070 uint64_t fence_value,
2071 unsigned int timeout_ms)
2072 {
2073 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
2074 struct device *dev = dqm->dev->adev->dev;
2075 uint64_t *fence_addr = dqm->fence_addr;
2076
2077 while (*fence_addr != fence_value) {
2078 /* Fatal err detected, this response won't come */
2079 if (amdgpu_amdkfd_is_fed(dqm->dev->adev))
2080 return -EIO;
2081
2082 if (time_after(jiffies, end_jiffies)) {
2083 dev_err(dev, "qcm fence wait loop timeout expired\n");
2084 /* In HWS case, this is used to halt the driver thread
2085 * in order not to mess up CP states before doing
2086 * scandumps for FW debugging.
2087 */
2088 while (halt_if_hws_hang)
2089 schedule();
2090
2091 return -ETIME;
2092 }
2093 schedule();
2094 }
2095
2096 return 0;
2097 }
2098
2099 /* dqm->lock mutex has to be locked before calling this function */
map_queues_cpsch(struct device_queue_manager * dqm)2100 static int map_queues_cpsch(struct device_queue_manager *dqm)
2101 {
2102 struct device *dev = dqm->dev->adev->dev;
2103 int retval;
2104
2105 if (!dqm->sched_running || dqm->sched_halt)
2106 return 0;
2107 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)
2108 return 0;
2109 if (dqm->active_runlist)
2110 return 0;
2111
2112 retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues);
2113 pr_debug("%s sent runlist\n", __func__);
2114 if (retval) {
2115 dev_err(dev, "failed to execute runlist\n");
2116 return retval;
2117 }
2118 dqm->active_runlist = true;
2119
2120 return retval;
2121 }
2122
set_queue_as_reset(struct device_queue_manager * dqm,struct queue * q,struct qcm_process_device * qpd)2123 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
2124 struct qcm_process_device *qpd)
2125 {
2126 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2127
2128 dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is reset\n",
2129 q->properties.queue_id, pdd->process->lead_thread->pid);
2130
2131 pdd->has_reset_queue = true;
2132 if (q->properties.is_active) {
2133 q->properties.is_active = false;
2134 decrement_queue_count(dqm, qpd, q);
2135 }
2136 }
2137
detect_queue_hang(struct device_queue_manager * dqm)2138 static int detect_queue_hang(struct device_queue_manager *dqm)
2139 {
2140 int i;
2141
2142 /* detect should be used only in dqm locked queue reset */
2143 if (WARN_ON(dqm->detect_hang_count > 0))
2144 return 0;
2145
2146 memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size);
2147
2148 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
2149 uint32_t mec, pipe, queue;
2150 int xcc_id;
2151
2152 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
2153 / dqm->dev->kfd->shared_resources.num_pipe_per_mec;
2154
2155 if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
2156 continue;
2157
2158 amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue);
2159
2160 for_each_inst(xcc_id, dqm->dev->xcc_mask) {
2161 uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr(
2162 dqm->dev->adev, pipe, queue, xcc_id);
2163 struct dqm_detect_hang_info hang_info;
2164
2165 if (!queue_addr)
2166 continue;
2167
2168 hang_info.pipe_id = pipe;
2169 hang_info.queue_id = queue;
2170 hang_info.xcc_id = xcc_id;
2171 hang_info.queue_address = queue_addr;
2172
2173 dqm->detect_hang_info[dqm->detect_hang_count] = hang_info;
2174 dqm->detect_hang_count++;
2175 }
2176 }
2177
2178 return dqm->detect_hang_count;
2179 }
2180
find_queue_by_address(struct device_queue_manager * dqm,uint64_t queue_address)2181 static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address)
2182 {
2183 struct device_process_node *cur;
2184 struct qcm_process_device *qpd;
2185 struct queue *q;
2186
2187 list_for_each_entry(cur, &dqm->queues, list) {
2188 qpd = cur->qpd;
2189 list_for_each_entry(q, &qpd->queues_list, list) {
2190 if (queue_address == q->properties.queue_address)
2191 return q;
2192 }
2193 }
2194
2195 return NULL;
2196 }
2197
reset_hung_queues(struct device_queue_manager * dqm)2198 static int reset_hung_queues(struct device_queue_manager *dqm)
2199 {
2200 int r = 0, reset_count = 0, i;
2201
2202 if (!dqm->detect_hang_info || dqm->is_hws_hang)
2203 return -EIO;
2204
2205 /* assume dqm locked. */
2206 if (!detect_queue_hang(dqm))
2207 return -ENOTRECOVERABLE;
2208
2209 for (i = 0; i < dqm->detect_hang_count; i++) {
2210 struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i];
2211 struct queue *q = find_queue_by_address(dqm, hang_info.queue_address);
2212 struct kfd_process_device *pdd;
2213 uint64_t queue_addr = 0;
2214
2215 if (!q) {
2216 r = -ENOTRECOVERABLE;
2217 goto reset_fail;
2218 }
2219
2220 pdd = kfd_get_process_device_data(dqm->dev, q->process);
2221 if (!pdd) {
2222 r = -ENOTRECOVERABLE;
2223 goto reset_fail;
2224 }
2225
2226 queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev,
2227 hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id,
2228 KFD_UNMAP_LATENCY_MS);
2229
2230 /* either reset failed or we reset an unexpected queue. */
2231 if (queue_addr != q->properties.queue_address) {
2232 r = -ENOTRECOVERABLE;
2233 goto reset_fail;
2234 }
2235
2236 set_queue_as_reset(dqm, q, &pdd->qpd);
2237 reset_count++;
2238 }
2239
2240 if (reset_count == dqm->detect_hang_count)
2241 kfd_signal_reset_event(dqm->dev);
2242 else
2243 r = -ENOTRECOVERABLE;
2244
2245 reset_fail:
2246 dqm->detect_hang_count = 0;
2247
2248 return r;
2249 }
2250
sdma_has_hang(struct device_queue_manager * dqm)2251 static bool sdma_has_hang(struct device_queue_manager *dqm)
2252 {
2253 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2254 int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2255 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2256 int i, j;
2257
2258 for (i = engine_start; i < engine_end; i++) {
2259 for (j = 0; j < num_queues_per_eng; j++) {
2260 if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j))
2261 continue;
2262
2263 return true;
2264 }
2265 }
2266
2267 return false;
2268 }
2269
set_sdma_queue_as_reset(struct device_queue_manager * dqm,uint32_t doorbell_off)2270 static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm,
2271 uint32_t doorbell_off)
2272 {
2273 struct device_process_node *cur;
2274 struct qcm_process_device *qpd;
2275 struct queue *q;
2276
2277 list_for_each_entry(cur, &dqm->queues, list) {
2278 qpd = cur->qpd;
2279 list_for_each_entry(q, &qpd->queues_list, list) {
2280 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2281 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) &&
2282 q->properties.doorbell_off == doorbell_off) {
2283 set_queue_as_reset(dqm, q, qpd);
2284 return true;
2285 }
2286 }
2287 }
2288
2289 return false;
2290 }
2291
reset_hung_queues_sdma(struct device_queue_manager * dqm)2292 static int reset_hung_queues_sdma(struct device_queue_manager *dqm)
2293 {
2294 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2295 int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2296 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2297 int r = 0, i, j;
2298
2299 if (dqm->is_hws_hang)
2300 return -EIO;
2301
2302 /* Scan for hung HW queues and reset engine. */
2303 dqm->detect_hang_count = 0;
2304 for (i = engine_start; i < engine_end; i++) {
2305 for (j = 0; j < num_queues_per_eng; j++) {
2306 uint32_t doorbell_off =
2307 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j);
2308
2309 if (!doorbell_off)
2310 continue;
2311
2312 /* Reset engine and check. */
2313 if (amdgpu_sdma_reset_engine(dqm->dev->adev, i) ||
2314 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) ||
2315 !set_sdma_queue_as_reset(dqm, doorbell_off)) {
2316 r = -ENOTRECOVERABLE;
2317 goto reset_fail;
2318 }
2319
2320 /* Should only expect one queue active per engine */
2321 dqm->detect_hang_count++;
2322 break;
2323 }
2324 }
2325
2326 /* Signal process reset */
2327 if (dqm->detect_hang_count)
2328 kfd_signal_reset_event(dqm->dev);
2329 else
2330 r = -ENOTRECOVERABLE;
2331
2332 reset_fail:
2333 dqm->detect_hang_count = 0;
2334
2335 return r;
2336 }
2337
reset_queues_on_hws_hang(struct device_queue_manager * dqm,bool is_sdma)2338 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma)
2339 {
2340 while (halt_if_hws_hang)
2341 schedule();
2342
2343 if (!amdgpu_gpu_recovery)
2344 return -ENOTRECOVERABLE;
2345
2346 return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm);
2347 }
2348
2349 /* dqm->lock mutex has to be locked before calling this function
2350 *
2351 * @grace_period: If USE_DEFAULT_GRACE_PERIOD then default wait time
2352 * for context switch latency. Lower values are used by debugger
2353 * since context switching are triggered at high frequency.
2354 * This is configured by setting CP_IQ_WAIT_TIME2.SCH_WAVE
2355 *
2356 */
unmap_queues_cpsch(struct device_queue_manager * dqm,enum kfd_unmap_queues_filter filter,uint32_t filter_param,uint32_t grace_period,bool reset)2357 static int unmap_queues_cpsch(struct device_queue_manager *dqm,
2358 enum kfd_unmap_queues_filter filter,
2359 uint32_t filter_param,
2360 uint32_t grace_period,
2361 bool reset)
2362 {
2363 struct device *dev = dqm->dev->adev->dev;
2364 struct mqd_manager *mqd_mgr;
2365 int retval;
2366
2367 if (!dqm->sched_running)
2368 return 0;
2369 if (!dqm->active_runlist)
2370 return 0;
2371 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem))
2372 return -EIO;
2373
2374 if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2375 retval = pm_config_dequeue_wait_counts(&dqm->packet_mgr,
2376 KFD_DEQUEUE_WAIT_SET_SCH_WAVE, grace_period);
2377 if (retval)
2378 goto out;
2379 }
2380
2381 retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset);
2382 if (retval)
2383 goto out;
2384
2385 *dqm->fence_addr = KFD_FENCE_INIT;
2386 mb();
2387 pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr,
2388 KFD_FENCE_COMPLETED);
2389 /* should be timed out */
2390 retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED,
2391 queue_preemption_timeout_ms);
2392 if (retval) {
2393 dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
2394 kfd_hws_hang(dqm);
2395 goto out;
2396 }
2397
2398 /* In the current MEC firmware implementation, if compute queue
2399 * doesn't response to the preemption request in time, HIQ will
2400 * abandon the unmap request without returning any timeout error
2401 * to driver. Instead, MEC firmware will log the doorbell of the
2402 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields.
2403 * To make sure the queue unmap was successful, driver need to
2404 * check those fields
2405 */
2406 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
2407 if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) &&
2408 reset_queues_on_hws_hang(dqm, false))
2409 goto reset_fail;
2410
2411 /* Check for SDMA hang and attempt SDMA reset */
2412 if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, true))
2413 goto reset_fail;
2414
2415 /* We need to reset the grace period value for this device */
2416 if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2417 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr,
2418 KFD_DEQUEUE_WAIT_RESET, 0 /* unused */))
2419 dev_err(dev, "Failed to reset grace period\n");
2420 }
2421
2422 pm_release_ib(&dqm->packet_mgr);
2423 dqm->active_runlist = false;
2424 out:
2425 up_read(&dqm->dev->adev->reset_domain->sem);
2426 return retval;
2427
2428 reset_fail:
2429 dqm->is_hws_hang = true;
2430 kfd_hws_hang(dqm);
2431 up_read(&dqm->dev->adev->reset_domain->sem);
2432 return -ETIME;
2433 }
2434
2435 /* only for compute queue */
reset_queues_cpsch(struct device_queue_manager * dqm,uint16_t pasid)2436 static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid)
2437 {
2438 int retval;
2439
2440 dqm_lock(dqm);
2441
2442 retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID,
2443 pasid, USE_DEFAULT_GRACE_PERIOD, true);
2444
2445 dqm_unlock(dqm);
2446 return retval;
2447 }
2448
2449 /* dqm->lock mutex has to be locked before calling this function */
execute_queues_cpsch(struct device_queue_manager * dqm,enum kfd_unmap_queues_filter filter,uint32_t filter_param,uint32_t grace_period)2450 static int execute_queues_cpsch(struct device_queue_manager *dqm,
2451 enum kfd_unmap_queues_filter filter,
2452 uint32_t filter_param,
2453 uint32_t grace_period)
2454 {
2455 int retval;
2456
2457 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem))
2458 return -EIO;
2459 retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false);
2460 if (!retval)
2461 retval = map_queues_cpsch(dqm);
2462 up_read(&dqm->dev->adev->reset_domain->sem);
2463 return retval;
2464 }
2465
wait_on_destroy_queue(struct device_queue_manager * dqm,struct queue * q)2466 static int wait_on_destroy_queue(struct device_queue_manager *dqm,
2467 struct queue *q)
2468 {
2469 struct kfd_process_device *pdd = kfd_get_process_device_data(q->device,
2470 q->process);
2471 int ret = 0;
2472
2473 if (WARN_ON(!pdd))
2474 return ret;
2475
2476 if (pdd->qpd.is_debug)
2477 return ret;
2478
2479 q->properties.is_being_destroyed = true;
2480
2481 if (pdd->process->debug_trap_enabled && q->properties.is_suspended) {
2482 dqm_unlock(dqm);
2483 mutex_unlock(&q->process->mutex);
2484 ret = wait_event_interruptible(dqm->destroy_wait,
2485 !q->properties.is_suspended);
2486
2487 mutex_lock(&q->process->mutex);
2488 dqm_lock(dqm);
2489 }
2490
2491 return ret;
2492 }
2493
destroy_queue_cpsch(struct device_queue_manager * dqm,struct qcm_process_device * qpd,struct queue * q)2494 static int destroy_queue_cpsch(struct device_queue_manager *dqm,
2495 struct qcm_process_device *qpd,
2496 struct queue *q)
2497 {
2498 int retval;
2499 struct mqd_manager *mqd_mgr;
2500 uint64_t sdma_val = 0;
2501 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2502 struct device *dev = dqm->dev->adev->dev;
2503
2504 /* Get the SDMA queue stats */
2505 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2506 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2507 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
2508 &sdma_val);
2509 if (retval)
2510 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
2511 q->properties.queue_id);
2512 }
2513
2514 /* remove queue from list to prevent rescheduling after preemption */
2515 dqm_lock(dqm);
2516
2517 retval = wait_on_destroy_queue(dqm, q);
2518
2519 if (retval) {
2520 dqm_unlock(dqm);
2521 return retval;
2522 }
2523
2524 if (qpd->is_debug) {
2525 /*
2526 * error, currently we do not allow to destroy a queue
2527 * of a currently debugged process
2528 */
2529 retval = -EBUSY;
2530 goto failed_try_destroy_debugged_queue;
2531
2532 }
2533
2534 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2535 q->properties.type)];
2536
2537 deallocate_doorbell(qpd, q);
2538
2539 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2540 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2541 deallocate_sdma_queue(dqm, q);
2542 pdd->sdma_past_activity_counter += sdma_val;
2543 }
2544
2545 if (q->properties.is_active) {
2546 decrement_queue_count(dqm, qpd, q);
2547 q->properties.is_active = false;
2548 if (!dqm->dev->kfd->shared_resources.enable_mes) {
2549 retval = execute_queues_cpsch(dqm,
2550 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
2551 USE_DEFAULT_GRACE_PERIOD);
2552 if (retval == -ETIME)
2553 qpd->reset_wavefronts = true;
2554 } else {
2555 retval = remove_queue_mes(dqm, q, qpd);
2556 }
2557 }
2558 list_del(&q->list);
2559 qpd->queue_count--;
2560
2561 /*
2562 * Unconditionally decrement this counter, regardless of the queue's
2563 * type
2564 */
2565 dqm->total_queue_count--;
2566 pr_debug("Total of %d queues are accountable so far\n",
2567 dqm->total_queue_count);
2568
2569 dqm_unlock(dqm);
2570
2571 /*
2572 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid
2573 * circular locking
2574 */
2575 kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE),
2576 qpd->pqm->process, q->device,
2577 -1, false, NULL, 0);
2578
2579 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2580
2581 return retval;
2582
2583 failed_try_destroy_debugged_queue:
2584
2585 dqm_unlock(dqm);
2586 return retval;
2587 }
2588
set_cache_memory_policy(struct device_queue_manager * dqm,struct qcm_process_device * qpd,enum cache_policy default_policy,enum cache_policy alternate_policy,void __user * alternate_aperture_base,uint64_t alternate_aperture_size,u32 misc_process_properties)2589 static bool set_cache_memory_policy(struct device_queue_manager *dqm,
2590 struct qcm_process_device *qpd,
2591 enum cache_policy default_policy,
2592 enum cache_policy alternate_policy,
2593 void __user *alternate_aperture_base,
2594 uint64_t alternate_aperture_size,
2595 u32 misc_process_properties)
2596 {
2597 bool retval = true;
2598
2599 if (!dqm->asic_ops.set_cache_memory_policy)
2600 return retval;
2601
2602 dqm_lock(dqm);
2603
2604 retval = dqm->asic_ops.set_cache_memory_policy(
2605 dqm,
2606 qpd,
2607 default_policy,
2608 alternate_policy,
2609 alternate_aperture_base,
2610 alternate_aperture_size,
2611 misc_process_properties);
2612
2613 if (retval)
2614 goto out;
2615
2616 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
2617 program_sh_mem_settings(dqm, qpd);
2618
2619 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
2620 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
2621 qpd->sh_mem_ape1_limit);
2622
2623 out:
2624 dqm_unlock(dqm);
2625 return retval;
2626 }
2627
process_termination_nocpsch(struct device_queue_manager * dqm,struct qcm_process_device * qpd)2628 static int process_termination_nocpsch(struct device_queue_manager *dqm,
2629 struct qcm_process_device *qpd)
2630 {
2631 struct queue *q;
2632 struct device_process_node *cur, *next_dpn;
2633 int retval = 0;
2634 bool found = false;
2635
2636 dqm_lock(dqm);
2637
2638 /* Clear all user mode queues */
2639 while (!list_empty(&qpd->queues_list)) {
2640 struct mqd_manager *mqd_mgr;
2641 int ret;
2642
2643 q = list_first_entry(&qpd->queues_list, struct queue, list);
2644 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2645 q->properties.type)];
2646 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
2647 if (ret)
2648 retval = ret;
2649 dqm_unlock(dqm);
2650 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2651 dqm_lock(dqm);
2652 }
2653
2654 /* Unregister process */
2655 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2656 if (qpd == cur->qpd) {
2657 list_del(&cur->list);
2658 kfree(cur);
2659 dqm->processes_count--;
2660 found = true;
2661 break;
2662 }
2663 }
2664
2665 dqm_unlock(dqm);
2666
2667 /* Outside the DQM lock because under the DQM lock we can't do
2668 * reclaim or take other locks that others hold while reclaiming.
2669 */
2670 if (found)
2671 kfd_dec_compute_active(dqm->dev);
2672
2673 return retval;
2674 }
2675
get_wave_state(struct device_queue_manager * dqm,struct queue * q,void __user * ctl_stack,u32 * ctl_stack_used_size,u32 * save_area_used_size)2676 static int get_wave_state(struct device_queue_manager *dqm,
2677 struct queue *q,
2678 void __user *ctl_stack,
2679 u32 *ctl_stack_used_size,
2680 u32 *save_area_used_size)
2681 {
2682 struct mqd_manager *mqd_mgr;
2683
2684 dqm_lock(dqm);
2685
2686 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
2687
2688 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
2689 q->properties.is_active || !q->device->kfd->cwsr_enabled ||
2690 !mqd_mgr->get_wave_state) {
2691 dqm_unlock(dqm);
2692 return -EINVAL;
2693 }
2694
2695 dqm_unlock(dqm);
2696
2697 /*
2698 * get_wave_state is outside the dqm lock to prevent circular locking
2699 * and the queue should be protected against destruction by the process
2700 * lock.
2701 */
2702 return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties,
2703 ctl_stack, ctl_stack_used_size, save_area_used_size);
2704 }
2705
get_queue_checkpoint_info(struct device_queue_manager * dqm,const struct queue * q,u32 * mqd_size,u32 * ctl_stack_size)2706 static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
2707 const struct queue *q,
2708 u32 *mqd_size,
2709 u32 *ctl_stack_size)
2710 {
2711 struct mqd_manager *mqd_mgr;
2712 enum KFD_MQD_TYPE mqd_type =
2713 get_mqd_type_from_queue_type(q->properties.type);
2714
2715 dqm_lock(dqm);
2716 mqd_mgr = dqm->mqd_mgrs[mqd_type];
2717 *mqd_size = mqd_mgr->mqd_size;
2718 *ctl_stack_size = 0;
2719
2720 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info)
2721 mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);
2722
2723 dqm_unlock(dqm);
2724 }
2725
checkpoint_mqd(struct device_queue_manager * dqm,const struct queue * q,void * mqd,void * ctl_stack)2726 static int checkpoint_mqd(struct device_queue_manager *dqm,
2727 const struct queue *q,
2728 void *mqd,
2729 void *ctl_stack)
2730 {
2731 struct mqd_manager *mqd_mgr;
2732 int r = 0;
2733 enum KFD_MQD_TYPE mqd_type =
2734 get_mqd_type_from_queue_type(q->properties.type);
2735
2736 dqm_lock(dqm);
2737
2738 if (q->properties.is_active || !q->device->kfd->cwsr_enabled) {
2739 r = -EINVAL;
2740 goto dqm_unlock;
2741 }
2742
2743 mqd_mgr = dqm->mqd_mgrs[mqd_type];
2744 if (!mqd_mgr->checkpoint_mqd) {
2745 r = -EOPNOTSUPP;
2746 goto dqm_unlock;
2747 }
2748
2749 mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack);
2750
2751 dqm_unlock:
2752 dqm_unlock(dqm);
2753 return r;
2754 }
2755
process_termination_cpsch(struct device_queue_manager * dqm,struct qcm_process_device * qpd)2756 static int process_termination_cpsch(struct device_queue_manager *dqm,
2757 struct qcm_process_device *qpd)
2758 {
2759 int retval;
2760 struct queue *q;
2761 struct device *dev = dqm->dev->adev->dev;
2762 struct kernel_queue *kq, *kq_next;
2763 struct mqd_manager *mqd_mgr;
2764 struct device_process_node *cur, *next_dpn;
2765 enum kfd_unmap_queues_filter filter =
2766 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
2767 bool found = false;
2768
2769 retval = 0;
2770
2771 dqm_lock(dqm);
2772
2773 /* Clean all kernel queues */
2774 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
2775 list_del(&kq->list);
2776 decrement_queue_count(dqm, qpd, kq->queue);
2777 qpd->is_debug = false;
2778 dqm->total_queue_count--;
2779 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
2780 }
2781
2782 /* Clear all user mode queues */
2783 list_for_each_entry(q, &qpd->queues_list, list) {
2784 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
2785 deallocate_sdma_queue(dqm, q);
2786 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
2787 deallocate_sdma_queue(dqm, q);
2788
2789 if (q->properties.is_active) {
2790 decrement_queue_count(dqm, qpd, q);
2791
2792 if (dqm->dev->kfd->shared_resources.enable_mes) {
2793 retval = remove_queue_mes(dqm, q, qpd);
2794 if (retval)
2795 dev_err(dev, "Failed to remove queue %d\n",
2796 q->properties.queue_id);
2797 }
2798 }
2799
2800 dqm->total_queue_count--;
2801 }
2802
2803 /* Unregister process */
2804 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2805 if (qpd == cur->qpd) {
2806 list_del(&cur->list);
2807 kfree(cur);
2808 dqm->processes_count--;
2809 found = true;
2810 break;
2811 }
2812 }
2813
2814 if (!dqm->dev->kfd->shared_resources.enable_mes)
2815 retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD);
2816
2817 if ((retval || qpd->reset_wavefronts) &&
2818 down_read_trylock(&dqm->dev->adev->reset_domain->sem)) {
2819 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
2820 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
2821 qpd->reset_wavefronts = false;
2822 up_read(&dqm->dev->adev->reset_domain->sem);
2823 }
2824
2825 /* Lastly, free mqd resources.
2826 * Do free_mqd() after dqm_unlock to avoid circular locking.
2827 */
2828 while (!list_empty(&qpd->queues_list)) {
2829 q = list_first_entry(&qpd->queues_list, struct queue, list);
2830 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2831 q->properties.type)];
2832 list_del(&q->list);
2833 qpd->queue_count--;
2834 dqm_unlock(dqm);
2835 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2836 dqm_lock(dqm);
2837 }
2838 dqm_unlock(dqm);
2839
2840 /* Outside the DQM lock because under the DQM lock we can't do
2841 * reclaim or take other locks that others hold while reclaiming.
2842 */
2843 if (found)
2844 kfd_dec_compute_active(dqm->dev);
2845
2846 return retval;
2847 }
2848
init_mqd_managers(struct device_queue_manager * dqm)2849 static int init_mqd_managers(struct device_queue_manager *dqm)
2850 {
2851 int i, j;
2852 struct device *dev = dqm->dev->adev->dev;
2853 struct mqd_manager *mqd_mgr;
2854
2855 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
2856 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
2857 if (!mqd_mgr) {
2858 dev_err(dev, "mqd manager [%d] initialization failed\n", i);
2859 goto out_free;
2860 }
2861 dqm->mqd_mgrs[i] = mqd_mgr;
2862 }
2863
2864 return 0;
2865
2866 out_free:
2867 for (j = 0; j < i; j++) {
2868 kfree(dqm->mqd_mgrs[j]);
2869 dqm->mqd_mgrs[j] = NULL;
2870 }
2871
2872 return -ENOMEM;
2873 }
2874
2875 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
allocate_hiq_sdma_mqd(struct device_queue_manager * dqm)2876 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
2877 {
2878 int retval;
2879 struct kfd_node *dev = dqm->dev;
2880 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
2881 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
2882 get_num_all_sdma_engines(dqm) *
2883 dev->kfd->device_info.num_sdma_queues_per_engine +
2884 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size *
2885 NUM_XCC(dqm->dev->xcc_mask));
2886
2887 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size,
2888 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr),
2889 (void *)&(mem_obj->cpu_ptr), false);
2890
2891 return retval;
2892 }
2893
device_queue_manager_init(struct kfd_node * dev)2894 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
2895 {
2896 struct device_queue_manager *dqm;
2897
2898 pr_debug("Loading device queue manager\n");
2899
2900 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
2901 if (!dqm)
2902 return NULL;
2903
2904 switch (dev->adev->asic_type) {
2905 /* HWS is not available on Hawaii. */
2906 case CHIP_HAWAII:
2907 /* HWS depends on CWSR for timely dequeue. CWSR is not
2908 * available on Tonga.
2909 *
2910 * FIXME: This argument also applies to Kaveri.
2911 */
2912 case CHIP_TONGA:
2913 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
2914 break;
2915 default:
2916 dqm->sched_policy = sched_policy;
2917 break;
2918 }
2919
2920 dqm->dev = dev;
2921 switch (dqm->sched_policy) {
2922 case KFD_SCHED_POLICY_HWS:
2923 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
2924 /* initialize dqm for cp scheduling */
2925 dqm->ops.create_queue = create_queue_cpsch;
2926 dqm->ops.initialize = initialize_cpsch;
2927 dqm->ops.start = start_cpsch;
2928 dqm->ops.stop = stop_cpsch;
2929 dqm->ops.halt = halt_cpsch;
2930 dqm->ops.unhalt = unhalt_cpsch;
2931 dqm->ops.destroy_queue = destroy_queue_cpsch;
2932 dqm->ops.update_queue = update_queue;
2933 dqm->ops.register_process = register_process;
2934 dqm->ops.unregister_process = unregister_process;
2935 dqm->ops.uninitialize = uninitialize;
2936 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
2937 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
2938 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2939 dqm->ops.process_termination = process_termination_cpsch;
2940 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
2941 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
2942 dqm->ops.get_wave_state = get_wave_state;
2943 dqm->ops.reset_queues = reset_queues_cpsch;
2944 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2945 dqm->ops.checkpoint_mqd = checkpoint_mqd;
2946 break;
2947 case KFD_SCHED_POLICY_NO_HWS:
2948 /* initialize dqm for no cp scheduling */
2949 dqm->ops.start = start_nocpsch;
2950 dqm->ops.stop = stop_nocpsch;
2951 dqm->ops.create_queue = create_queue_nocpsch;
2952 dqm->ops.destroy_queue = destroy_queue_nocpsch;
2953 dqm->ops.update_queue = update_queue;
2954 dqm->ops.register_process = register_process;
2955 dqm->ops.unregister_process = unregister_process;
2956 dqm->ops.initialize = initialize_nocpsch;
2957 dqm->ops.uninitialize = uninitialize;
2958 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2959 dqm->ops.process_termination = process_termination_nocpsch;
2960 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
2961 dqm->ops.restore_process_queues =
2962 restore_process_queues_nocpsch;
2963 dqm->ops.get_wave_state = get_wave_state;
2964 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2965 dqm->ops.checkpoint_mqd = checkpoint_mqd;
2966 break;
2967 default:
2968 dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy);
2969 goto out_free;
2970 }
2971
2972 switch (dev->adev->asic_type) {
2973 case CHIP_KAVERI:
2974 case CHIP_HAWAII:
2975 device_queue_manager_init_cik(&dqm->asic_ops);
2976 break;
2977
2978 case CHIP_CARRIZO:
2979 case CHIP_TONGA:
2980 case CHIP_FIJI:
2981 case CHIP_POLARIS10:
2982 case CHIP_POLARIS11:
2983 case CHIP_POLARIS12:
2984 case CHIP_VEGAM:
2985 device_queue_manager_init_vi(&dqm->asic_ops);
2986 break;
2987
2988 default:
2989 if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0))
2990 device_queue_manager_init_v12(&dqm->asic_ops);
2991 else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0))
2992 device_queue_manager_init_v11(&dqm->asic_ops);
2993 else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
2994 device_queue_manager_init_v10(&dqm->asic_ops);
2995 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
2996 device_queue_manager_init_v9(&dqm->asic_ops);
2997 else {
2998 WARN(1, "Unexpected ASIC family %u",
2999 dev->adev->asic_type);
3000 goto out_free;
3001 }
3002 }
3003
3004 if (init_mqd_managers(dqm))
3005 goto out_free;
3006
3007 if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) {
3008 dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n");
3009 goto out_free;
3010 }
3011
3012 if (!dqm->ops.initialize(dqm)) {
3013 init_waitqueue_head(&dqm->destroy_wait);
3014 return dqm;
3015 }
3016
3017 out_free:
3018 kfree(dqm);
3019 return NULL;
3020 }
3021
deallocate_hiq_sdma_mqd(struct kfd_node * dev,struct kfd_mem_obj * mqd)3022 static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
3023 struct kfd_mem_obj *mqd)
3024 {
3025 WARN(!mqd, "No hiq sdma mqd trunk to free");
3026
3027 amdgpu_amdkfd_free_gtt_mem(dev->adev, &mqd->gtt_mem);
3028 }
3029
device_queue_manager_uninit(struct device_queue_manager * dqm)3030 void device_queue_manager_uninit(struct device_queue_manager *dqm)
3031 {
3032 dqm->ops.stop(dqm);
3033 dqm->ops.uninitialize(dqm);
3034 if (!dqm->dev->kfd->shared_resources.enable_mes)
3035 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
3036 kfree(dqm);
3037 }
3038
kfd_dqm_suspend_bad_queue_mes(struct kfd_node * knode,u32 pasid,u32 doorbell_id)3039 int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id)
3040 {
3041 struct kfd_process_device *pdd = NULL;
3042 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd);
3043 struct device_queue_manager *dqm = knode->dqm;
3044 struct device *dev = dqm->dev->adev->dev;
3045 struct qcm_process_device *qpd;
3046 struct queue *q = NULL;
3047 int ret = 0;
3048
3049 if (!pdd)
3050 return -EINVAL;
3051
3052 dqm_lock(dqm);
3053
3054 if (pdd) {
3055 qpd = &pdd->qpd;
3056
3057 list_for_each_entry(q, &qpd->queues_list, list) {
3058 if (q->doorbell_id == doorbell_id && q->properties.is_active) {
3059 ret = suspend_all_queues_mes(dqm);
3060 if (ret) {
3061 dev_err(dev, "Suspending all queues failed");
3062 goto out;
3063 }
3064
3065 q->properties.is_evicted = true;
3066 q->properties.is_active = false;
3067 decrement_queue_count(dqm, qpd, q);
3068
3069 ret = remove_queue_mes(dqm, q, qpd);
3070 if (ret) {
3071 dev_err(dev, "Removing bad queue failed");
3072 goto out;
3073 }
3074
3075 ret = resume_all_queues_mes(dqm);
3076 if (ret)
3077 dev_err(dev, "Resuming all queues failed");
3078
3079 break;
3080 }
3081 }
3082 }
3083
3084 out:
3085 dqm_unlock(dqm);
3086 kfd_unref_process(p);
3087 return ret;
3088 }
3089
kfd_dqm_evict_pasid_mes(struct device_queue_manager * dqm,struct qcm_process_device * qpd)3090 static int kfd_dqm_evict_pasid_mes(struct device_queue_manager *dqm,
3091 struct qcm_process_device *qpd)
3092 {
3093 struct device *dev = dqm->dev->adev->dev;
3094 int ret = 0;
3095
3096 /* Check if process is already evicted */
3097 dqm_lock(dqm);
3098 if (qpd->evicted) {
3099 /* Increment the evicted count to make sure the
3100 * process stays evicted before its terminated.
3101 */
3102 qpd->evicted++;
3103 dqm_unlock(dqm);
3104 goto out;
3105 }
3106 dqm_unlock(dqm);
3107
3108 ret = suspend_all_queues_mes(dqm);
3109 if (ret) {
3110 dev_err(dev, "Suspending all queues failed");
3111 goto out;
3112 }
3113
3114 ret = dqm->ops.evict_process_queues(dqm, qpd);
3115 if (ret) {
3116 dev_err(dev, "Evicting process queues failed");
3117 goto out;
3118 }
3119
3120 ret = resume_all_queues_mes(dqm);
3121 if (ret)
3122 dev_err(dev, "Resuming all queues failed");
3123
3124 out:
3125 return ret;
3126 }
3127
kfd_evict_process_device(struct kfd_process_device * pdd)3128 int kfd_evict_process_device(struct kfd_process_device *pdd)
3129 {
3130 struct device_queue_manager *dqm;
3131 struct kfd_process *p;
3132 int ret = 0;
3133
3134 p = pdd->process;
3135 dqm = pdd->dev->dqm;
3136
3137 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
3138
3139 if (dqm->dev->kfd->shared_resources.enable_mes)
3140 ret = kfd_dqm_evict_pasid_mes(dqm, &pdd->qpd);
3141 else
3142 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
3143
3144 return ret;
3145 }
3146
reserve_debug_trap_vmid(struct device_queue_manager * dqm,struct qcm_process_device * qpd)3147 int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
3148 struct qcm_process_device *qpd)
3149 {
3150 int r;
3151 struct device *dev = dqm->dev->adev->dev;
3152 int updated_vmid_mask;
3153
3154 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3155 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3156 return -EINVAL;
3157 }
3158
3159 dqm_lock(dqm);
3160
3161 if (dqm->trap_debug_vmid != 0) {
3162 dev_err(dev, "Trap debug id already reserved\n");
3163 r = -EBUSY;
3164 goto out_unlock;
3165 }
3166
3167 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
3168 USE_DEFAULT_GRACE_PERIOD, false);
3169 if (r)
3170 goto out_unlock;
3171
3172 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3173 updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd);
3174
3175 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3176 dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd;
3177 r = set_sched_resources(dqm);
3178 if (r)
3179 goto out_unlock;
3180
3181 r = map_queues_cpsch(dqm);
3182 if (r)
3183 goto out_unlock;
3184
3185 pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid);
3186
3187 out_unlock:
3188 dqm_unlock(dqm);
3189 return r;
3190 }
3191
3192 /*
3193 * Releases vmid for the trap debugger
3194 */
release_debug_trap_vmid(struct device_queue_manager * dqm,struct qcm_process_device * qpd)3195 int release_debug_trap_vmid(struct device_queue_manager *dqm,
3196 struct qcm_process_device *qpd)
3197 {
3198 struct device *dev = dqm->dev->adev->dev;
3199 int r;
3200 int updated_vmid_mask;
3201 uint32_t trap_debug_vmid;
3202
3203 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3204 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3205 return -EINVAL;
3206 }
3207
3208 dqm_lock(dqm);
3209 trap_debug_vmid = dqm->trap_debug_vmid;
3210 if (dqm->trap_debug_vmid == 0) {
3211 dev_err(dev, "Trap debug id is not reserved\n");
3212 r = -EINVAL;
3213 goto out_unlock;
3214 }
3215
3216 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
3217 USE_DEFAULT_GRACE_PERIOD, false);
3218 if (r)
3219 goto out_unlock;
3220
3221 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3222 updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd);
3223
3224 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3225 dqm->trap_debug_vmid = 0;
3226 r = set_sched_resources(dqm);
3227 if (r)
3228 goto out_unlock;
3229
3230 r = map_queues_cpsch(dqm);
3231 if (r)
3232 goto out_unlock;
3233
3234 pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid);
3235
3236 out_unlock:
3237 dqm_unlock(dqm);
3238 return r;
3239 }
3240
3241 #define QUEUE_NOT_FOUND -1
3242 /* invalidate queue operation in array */
q_array_invalidate(uint32_t num_queues,uint32_t * queue_ids)3243 static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids)
3244 {
3245 int i;
3246
3247 for (i = 0; i < num_queues; i++)
3248 queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK;
3249 }
3250
3251 /* find queue index in array */
q_array_get_index(unsigned int queue_id,uint32_t num_queues,uint32_t * queue_ids)3252 static int q_array_get_index(unsigned int queue_id,
3253 uint32_t num_queues,
3254 uint32_t *queue_ids)
3255 {
3256 int i;
3257
3258 for (i = 0; i < num_queues; i++)
3259 if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK))
3260 return i;
3261
3262 return QUEUE_NOT_FOUND;
3263 }
3264
3265 struct copy_context_work_handler_workarea {
3266 struct work_struct copy_context_work;
3267 struct kfd_process *p;
3268 };
3269
copy_context_work_handler(struct work_struct * work)3270 static void copy_context_work_handler(struct work_struct *work)
3271 {
3272 struct copy_context_work_handler_workarea *workarea;
3273 struct mqd_manager *mqd_mgr;
3274 struct queue *q;
3275 struct mm_struct *mm;
3276 struct kfd_process *p;
3277 uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size;
3278 int i;
3279
3280 workarea = container_of(work,
3281 struct copy_context_work_handler_workarea,
3282 copy_context_work);
3283
3284 p = workarea->p;
3285 mm = get_task_mm(p->lead_thread);
3286
3287 if (!mm)
3288 return;
3289
3290 kthread_use_mm(mm);
3291 for (i = 0; i < p->n_pdds; i++) {
3292 struct kfd_process_device *pdd = p->pdds[i];
3293 struct device_queue_manager *dqm = pdd->dev->dqm;
3294 struct qcm_process_device *qpd = &pdd->qpd;
3295
3296 list_for_each_entry(q, &qpd->queues_list, list) {
3297 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE)
3298 continue;
3299
3300 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
3301
3302 /* We ignore the return value from get_wave_state
3303 * because
3304 * i) right now, it always returns 0, and
3305 * ii) if we hit an error, we would continue to the
3306 * next queue anyway.
3307 */
3308 mqd_mgr->get_wave_state(mqd_mgr,
3309 q->mqd,
3310 &q->properties,
3311 (void __user *) q->properties.ctx_save_restore_area_address,
3312 &tmp_ctl_stack_used_size,
3313 &tmp_save_area_used_size);
3314 }
3315 }
3316 kthread_unuse_mm(mm);
3317 mmput(mm);
3318 }
3319
get_queue_ids(uint32_t num_queues,uint32_t * usr_queue_id_array)3320 static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array)
3321 {
3322 size_t array_size = num_queues * sizeof(uint32_t);
3323
3324 if (!usr_queue_id_array)
3325 return NULL;
3326
3327 return memdup_user(usr_queue_id_array, array_size);
3328 }
3329
resume_queues(struct kfd_process * p,uint32_t num_queues,uint32_t * usr_queue_id_array)3330 int resume_queues(struct kfd_process *p,
3331 uint32_t num_queues,
3332 uint32_t *usr_queue_id_array)
3333 {
3334 uint32_t *queue_ids = NULL;
3335 int total_resumed = 0;
3336 int i;
3337
3338 if (usr_queue_id_array) {
3339 queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3340
3341 if (IS_ERR(queue_ids))
3342 return PTR_ERR(queue_ids);
3343
3344 /* mask all queues as invalid. unmask per successful request */
3345 q_array_invalidate(num_queues, queue_ids);
3346 }
3347
3348 for (i = 0; i < p->n_pdds; i++) {
3349 struct kfd_process_device *pdd = p->pdds[i];
3350 struct device_queue_manager *dqm = pdd->dev->dqm;
3351 struct device *dev = dqm->dev->adev->dev;
3352 struct qcm_process_device *qpd = &pdd->qpd;
3353 struct queue *q;
3354 int r, per_device_resumed = 0;
3355
3356 dqm_lock(dqm);
3357
3358 /* unmask queues that resume or already resumed as valid */
3359 list_for_each_entry(q, &qpd->queues_list, list) {
3360 int q_idx = QUEUE_NOT_FOUND;
3361
3362 if (queue_ids)
3363 q_idx = q_array_get_index(
3364 q->properties.queue_id,
3365 num_queues,
3366 queue_ids);
3367
3368 if (!queue_ids || q_idx != QUEUE_NOT_FOUND) {
3369 int err = resume_single_queue(dqm, &pdd->qpd, q);
3370
3371 if (queue_ids) {
3372 if (!err) {
3373 queue_ids[q_idx] &=
3374 ~KFD_DBG_QUEUE_INVALID_MASK;
3375 } else {
3376 queue_ids[q_idx] |=
3377 KFD_DBG_QUEUE_ERROR_MASK;
3378 break;
3379 }
3380 }
3381
3382 if (dqm->dev->kfd->shared_resources.enable_mes) {
3383 wake_up_all(&dqm->destroy_wait);
3384 if (!err)
3385 total_resumed++;
3386 } else {
3387 per_device_resumed++;
3388 }
3389 }
3390 }
3391
3392 if (!per_device_resumed) {
3393 dqm_unlock(dqm);
3394 continue;
3395 }
3396
3397 r = execute_queues_cpsch(dqm,
3398 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
3399 0,
3400 USE_DEFAULT_GRACE_PERIOD);
3401 if (r) {
3402 dev_err(dev, "Failed to resume process queues\n");
3403 if (queue_ids) {
3404 list_for_each_entry(q, &qpd->queues_list, list) {
3405 int q_idx = q_array_get_index(
3406 q->properties.queue_id,
3407 num_queues,
3408 queue_ids);
3409
3410 /* mask queue as error on resume fail */
3411 if (q_idx != QUEUE_NOT_FOUND)
3412 queue_ids[q_idx] |=
3413 KFD_DBG_QUEUE_ERROR_MASK;
3414 }
3415 }
3416 } else {
3417 wake_up_all(&dqm->destroy_wait);
3418 total_resumed += per_device_resumed;
3419 }
3420
3421 dqm_unlock(dqm);
3422 }
3423
3424 if (queue_ids) {
3425 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,
3426 num_queues * sizeof(uint32_t)))
3427 pr_err("copy_to_user failed on queue resume\n");
3428
3429 kfree(queue_ids);
3430 }
3431
3432 return total_resumed;
3433 }
3434
suspend_queues(struct kfd_process * p,uint32_t num_queues,uint32_t grace_period,uint64_t exception_clear_mask,uint32_t * usr_queue_id_array)3435 int suspend_queues(struct kfd_process *p,
3436 uint32_t num_queues,
3437 uint32_t grace_period,
3438 uint64_t exception_clear_mask,
3439 uint32_t *usr_queue_id_array)
3440 {
3441 uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3442 int total_suspended = 0;
3443 int i;
3444
3445 if (IS_ERR(queue_ids))
3446 return PTR_ERR(queue_ids);
3447
3448 /* mask all queues as invalid. umask on successful request */
3449 q_array_invalidate(num_queues, queue_ids);
3450
3451 for (i = 0; i < p->n_pdds; i++) {
3452 struct kfd_process_device *pdd = p->pdds[i];
3453 struct device_queue_manager *dqm = pdd->dev->dqm;
3454 struct device *dev = dqm->dev->adev->dev;
3455 struct qcm_process_device *qpd = &pdd->qpd;
3456 struct queue *q;
3457 int r, per_device_suspended = 0;
3458
3459 mutex_lock(&p->event_mutex);
3460 dqm_lock(dqm);
3461
3462 /* unmask queues that suspend or already suspended */
3463 list_for_each_entry(q, &qpd->queues_list, list) {
3464 int q_idx = q_array_get_index(q->properties.queue_id,
3465 num_queues,
3466 queue_ids);
3467
3468 if (q_idx != QUEUE_NOT_FOUND) {
3469 int err = suspend_single_queue(dqm, pdd, q);
3470 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes;
3471
3472 if (!err) {
3473 queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK;
3474 if (exception_clear_mask && is_mes)
3475 q->properties.exception_status &=
3476 ~exception_clear_mask;
3477
3478 if (is_mes)
3479 total_suspended++;
3480 else
3481 per_device_suspended++;
3482 } else if (err != -EBUSY) {
3483 r = err;
3484 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3485 break;
3486 }
3487 }
3488 }
3489
3490 if (!per_device_suspended) {
3491 dqm_unlock(dqm);
3492 mutex_unlock(&p->event_mutex);
3493 if (total_suspended)
3494 amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev);
3495 continue;
3496 }
3497
3498 r = execute_queues_cpsch(dqm,
3499 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
3500 grace_period);
3501
3502 if (r)
3503 dev_err(dev, "Failed to suspend process queues.\n");
3504 else
3505 total_suspended += per_device_suspended;
3506
3507 list_for_each_entry(q, &qpd->queues_list, list) {
3508 int q_idx = q_array_get_index(q->properties.queue_id,
3509 num_queues, queue_ids);
3510
3511 if (q_idx == QUEUE_NOT_FOUND)
3512 continue;
3513
3514 /* mask queue as error on suspend fail */
3515 if (r)
3516 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3517 else if (exception_clear_mask)
3518 q->properties.exception_status &=
3519 ~exception_clear_mask;
3520 }
3521
3522 dqm_unlock(dqm);
3523 mutex_unlock(&p->event_mutex);
3524 amdgpu_device_flush_hdp(dqm->dev->adev, NULL);
3525 }
3526
3527 if (total_suspended) {
3528 struct copy_context_work_handler_workarea copy_context_worker;
3529
3530 INIT_WORK_ONSTACK(
3531 ©_context_worker.copy_context_work,
3532 copy_context_work_handler);
3533
3534 copy_context_worker.p = p;
3535
3536 schedule_work(©_context_worker.copy_context_work);
3537
3538
3539 flush_work(©_context_worker.copy_context_work);
3540 destroy_work_on_stack(©_context_worker.copy_context_work);
3541 }
3542
3543 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,
3544 num_queues * sizeof(uint32_t)))
3545 pr_err("copy_to_user failed on queue suspend\n");
3546
3547 kfree(queue_ids);
3548
3549 return total_suspended;
3550 }
3551
set_queue_type_for_user(struct queue_properties * q_props)3552 static uint32_t set_queue_type_for_user(struct queue_properties *q_props)
3553 {
3554 switch (q_props->type) {
3555 case KFD_QUEUE_TYPE_COMPUTE:
3556 return q_props->format == KFD_QUEUE_FORMAT_PM4
3557 ? KFD_IOC_QUEUE_TYPE_COMPUTE
3558 : KFD_IOC_QUEUE_TYPE_COMPUTE_AQL;
3559 case KFD_QUEUE_TYPE_SDMA:
3560 return KFD_IOC_QUEUE_TYPE_SDMA;
3561 case KFD_QUEUE_TYPE_SDMA_XGMI:
3562 return KFD_IOC_QUEUE_TYPE_SDMA_XGMI;
3563 default:
3564 WARN_ONCE(true, "queue type not recognized!");
3565 return 0xffffffff;
3566 };
3567 }
3568
set_queue_snapshot_entry(struct queue * q,uint64_t exception_clear_mask,struct kfd_queue_snapshot_entry * qss_entry)3569 void set_queue_snapshot_entry(struct queue *q,
3570 uint64_t exception_clear_mask,
3571 struct kfd_queue_snapshot_entry *qss_entry)
3572 {
3573 qss_entry->ring_base_address = q->properties.queue_address;
3574 qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr;
3575 qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr;
3576 qss_entry->ctx_save_restore_address =
3577 q->properties.ctx_save_restore_area_address;
3578 qss_entry->ctx_save_restore_area_size =
3579 q->properties.ctx_save_restore_area_size;
3580 qss_entry->exception_status = q->properties.exception_status;
3581 qss_entry->queue_id = q->properties.queue_id;
3582 qss_entry->gpu_id = q->device->id;
3583 qss_entry->ring_size = (uint32_t)q->properties.queue_size;
3584 qss_entry->queue_type = set_queue_type_for_user(&q->properties);
3585 q->properties.exception_status &= ~exception_clear_mask;
3586 }
3587
debug_lock_and_unmap(struct device_queue_manager * dqm)3588 int debug_lock_and_unmap(struct device_queue_manager *dqm)
3589 {
3590 struct device *dev = dqm->dev->adev->dev;
3591 int r;
3592
3593 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3594 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3595 return -EINVAL;
3596 }
3597
3598 if (!kfd_dbg_is_per_vmid_supported(dqm->dev))
3599 return 0;
3600
3601 dqm_lock(dqm);
3602
3603 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false);
3604 if (r)
3605 dqm_unlock(dqm);
3606
3607 return r;
3608 }
3609
debug_map_and_unlock(struct device_queue_manager * dqm)3610 int debug_map_and_unlock(struct device_queue_manager *dqm)
3611 {
3612 struct device *dev = dqm->dev->adev->dev;
3613 int r;
3614
3615 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3616 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3617 return -EINVAL;
3618 }
3619
3620 if (!kfd_dbg_is_per_vmid_supported(dqm->dev))
3621 return 0;
3622
3623 r = map_queues_cpsch(dqm);
3624
3625 dqm_unlock(dqm);
3626
3627 return r;
3628 }
3629
debug_refresh_runlist(struct device_queue_manager * dqm)3630 int debug_refresh_runlist(struct device_queue_manager *dqm)
3631 {
3632 int r = debug_lock_and_unmap(dqm);
3633
3634 if (r)
3635 return r;
3636
3637 return debug_map_and_unlock(dqm);
3638 }
3639
kfd_dqm_is_queue_in_process(struct device_queue_manager * dqm,struct qcm_process_device * qpd,int doorbell_off,u32 * queue_format)3640 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,
3641 struct qcm_process_device *qpd,
3642 int doorbell_off, u32 *queue_format)
3643 {
3644 struct queue *q;
3645 bool r = false;
3646
3647 if (!queue_format)
3648 return r;
3649
3650 dqm_lock(dqm);
3651
3652 list_for_each_entry(q, &qpd->queues_list, list) {
3653 if (q->properties.doorbell_off == doorbell_off) {
3654 *queue_format = q->properties.format;
3655 r = true;
3656 goto out;
3657 }
3658 }
3659
3660 out:
3661 dqm_unlock(dqm);
3662 return r;
3663 }
3664 #if defined(CONFIG_DEBUG_FS)
3665
seq_reg_dump(struct seq_file * m,uint32_t (* dump)[2],uint32_t n_regs)3666 static void seq_reg_dump(struct seq_file *m,
3667 uint32_t (*dump)[2], uint32_t n_regs)
3668 {
3669 uint32_t i, count;
3670
3671 for (i = 0, count = 0; i < n_regs; i++) {
3672 if (count == 0 ||
3673 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
3674 seq_printf(m, "%s %08x: %08x",
3675 i ? "\n" : "",
3676 dump[i][0], dump[i][1]);
3677 count = 7;
3678 } else {
3679 seq_printf(m, " %08x", dump[i][1]);
3680 count--;
3681 }
3682 }
3683
3684 seq_puts(m, "\n");
3685 }
3686
dqm_debugfs_hqds(struct seq_file * m,void * data)3687 int dqm_debugfs_hqds(struct seq_file *m, void *data)
3688 {
3689 struct device_queue_manager *dqm = data;
3690 uint32_t xcc_mask = dqm->dev->xcc_mask;
3691 uint32_t (*dump)[2], n_regs;
3692 int pipe, queue;
3693 int r = 0, xcc_id;
3694 uint32_t sdma_engine_start;
3695
3696 if (!dqm->sched_running) {
3697 seq_puts(m, " Device is stopped\n");
3698 return 0;
3699 }
3700
3701 for_each_inst(xcc_id, xcc_mask) {
3702 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3703 KFD_CIK_HIQ_PIPE,
3704 KFD_CIK_HIQ_QUEUE, &dump,
3705 &n_regs, xcc_id);
3706 if (!r) {
3707 seq_printf(
3708 m,
3709 " Inst %d, HIQ on MEC %d Pipe %d Queue %d\n",
3710 xcc_id,
3711 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1,
3712 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm),
3713 KFD_CIK_HIQ_QUEUE);
3714 seq_reg_dump(m, dump, n_regs);
3715
3716 kfree(dump);
3717 }
3718
3719 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
3720 int pipe_offset = pipe * get_queues_per_pipe(dqm);
3721
3722 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
3723 if (!test_bit(pipe_offset + queue,
3724 dqm->dev->kfd->shared_resources.cp_queue_bitmap))
3725 continue;
3726
3727 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3728 pipe, queue,
3729 &dump, &n_regs,
3730 xcc_id);
3731 if (r)
3732 break;
3733
3734 seq_printf(m,
3735 " Inst %d, CP Pipe %d, Queue %d\n",
3736 xcc_id, pipe, queue);
3737 seq_reg_dump(m, dump, n_regs);
3738
3739 kfree(dump);
3740 }
3741 }
3742 }
3743
3744 sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
3745 for (pipe = sdma_engine_start;
3746 pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm));
3747 pipe++) {
3748 for (queue = 0;
3749 queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
3750 queue++) {
3751 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
3752 dqm->dev->adev, pipe, queue, &dump, &n_regs);
3753 if (r)
3754 break;
3755
3756 seq_printf(m, " SDMA Engine %d, RLC %d\n",
3757 pipe, queue);
3758 seq_reg_dump(m, dump, n_regs);
3759
3760 kfree(dump);
3761 }
3762 }
3763
3764 return r;
3765 }
3766
dqm_debugfs_hang_hws(struct device_queue_manager * dqm)3767 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm)
3768 {
3769 int r = 0;
3770
3771 dqm_lock(dqm);
3772 r = pm_debugfs_hang_hws(&dqm->packet_mgr);
3773 if (r) {
3774 dqm_unlock(dqm);
3775 return r;
3776 }
3777 dqm->active_runlist = true;
3778 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
3779 0, USE_DEFAULT_GRACE_PERIOD);
3780 dqm_unlock(dqm);
3781
3782 return r;
3783 }
3784
3785 #endif
3786