projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: Add H/W debugger kfd->kgd functions
[deliverable/linux.git] / drivers / gpu / drm / radeon / radeon_kfd.c
CommitLineData
e28740ec
OG		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/module.h>
24#include <linux/fdtable.h>
25#include <linux/uaccess.h>
26#include <drm/drmP.h>
27#include "radeon.h"
28#include "cikd.h"
29#include "cik_reg.h"
30#include "radeon_kfd.h"
f7694323
OG		 31#include "radeon_ucode.h"
32#include <linux/firmware.h>
836aabc0 33#include "cik_structs.h"
e28740ec
OG		 34
35#define CIK_PIPE_PER_MEC (4)
36
a6186f4d
YS		 37static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = {
38 TCP_WATCH0_ADDR_H, TCP_WATCH0_ADDR_L, TCP_WATCH0_CNTL,
39 TCP_WATCH1_ADDR_H, TCP_WATCH1_ADDR_L, TCP_WATCH1_CNTL,
40 TCP_WATCH2_ADDR_H, TCP_WATCH2_ADDR_L, TCP_WATCH2_CNTL,
41 TCP_WATCH3_ADDR_H, TCP_WATCH3_ADDR_L, TCP_WATCH3_CNTL
42};
43
e28740ec 44struct kgd_mem {
ceae881b 45 struct radeon_bo *bo;
632aa2cb 46 uint64_t gpu_addr;
ceae881b 47 void *cpu_ptr;
e28740ec
OG		 48};
49
e28740ec 50
ceae881b
OG		 51static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
52 void **mem_obj, uint64_t *gpu_addr,
53 void **cpu_ptr);
54
55static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
56
e28740ec
OG		 57static uint64_t get_vmem_size(struct kgd_dev *kgd);
58static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
59
60static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
f7694323 61static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
e28740ec
OG		 62
63/*
64 * Register access functions
65 */
66
67static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
68 uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
69 uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
70
71static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
72 unsigned int vmid);
73
e28740ec
OG		 74static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
75 uint32_t hpd_size, uint64_t hpd_gpu_addr);
d36b94fc 76static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
e28740ec
OG		 77static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
78 uint32_t queue_id, uint32_t __user *wptr);
a84a9903 79static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
b64b8afc 80static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
e28740ec
OG		 81 uint32_t pipe_id, uint32_t queue_id);
82
83static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
84 unsigned int timeout, uint32_t pipe_id,
85 uint32_t queue_id);
a84a9903
BG		 86static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
87static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
88 unsigned int timeout);
a6186f4d
YS		 89static int kgd_address_watch_disable(struct kgd_dev *kgd);
90static int kgd_address_watch_execute(struct kgd_dev *kgd,
91 unsigned int watch_point_id,
92 uint32_t cntl_val,
93 uint32_t addr_hi,
94 uint32_t addr_lo);
95static int kgd_wave_control_execute(struct kgd_dev *kgd,
96 uint32_t gfx_index_val,
97 uint32_t sq_cmd);
98static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
99 unsigned int watch_point_id,
100 unsigned int reg_offset);
e28740ec
OG		 101
102static const struct kfd2kgd_calls kfd2kgd = {
ceae881b
OG		 103 .init_gtt_mem_allocation = alloc_gtt_mem,
104 .free_gtt_mem = free_gtt_mem,
e28740ec
OG		 105 .get_vmem_size = get_vmem_size,
106 .get_gpu_clock_counter = get_gpu_clock_counter,
107 .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
108 .program_sh_mem_settings = kgd_program_sh_mem_settings,
109 .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
e28740ec 110 .init_pipeline = kgd_init_pipeline,
d36b94fc 111 .init_interrupts = kgd_init_interrupts,
e28740ec 112 .hqd_load = kgd_hqd_load,
a84a9903 113 .hqd_sdma_load = kgd_hqd_sdma_load,
b64b8afc 114 .hqd_is_occupied = kgd_hqd_is_occupied,
a84a9903 115 .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
e28740ec 116 .hqd_destroy = kgd_hqd_destroy,
a84a9903 117 .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
a6186f4d
YS		 118 .address_watch_disable = kgd_address_watch_disable,
119 .address_watch_execute = kgd_address_watch_execute,
120 .wave_control_execute = kgd_wave_control_execute,
121 .address_watch_get_offset = kgd_address_watch_get_offset,
f7694323 122 .get_fw_version = get_fw_version
e28740ec
OG		 123};
124
125static const struct kgd2kfd_calls *kgd2kfd;
126
127bool radeon_kfd_init(void)
128{
38c2adfb 129#if defined(CONFIG_HSA_AMD_MODULE)
cea405b1 130 bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
e28740ec
OG		 131
132 kgd2kfd_init_p = symbol_request(kgd2kfd_init);
133
134 if (kgd2kfd_init_p == NULL)
135 return false;
136
cea405b1 137 if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd)) {
e28740ec
OG		 138 symbol_put(kgd2kfd_init);
139 kgd2kfd = NULL;
140
141 return false;
142 }
143
144 return true;
38c2adfb 145#elif defined(CONFIG_HSA_AMD)
cea405b1 146 if (!kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd)) {
38c2adfb
OG		 147 kgd2kfd = NULL;
148
149 return false;
150 }
151
152 return true;
153#else
154 return false;
155#endif
e28740ec
OG		 156}
157
158void radeon_kfd_fini(void)
159{
160 if (kgd2kfd) {
161 kgd2kfd->exit();
162 symbol_put(kgd2kfd_init);
163 }
164}
165
166void radeon_kfd_device_probe(struct radeon_device *rdev)
167{
168 if (kgd2kfd)
cea405b1
XZ		 169 rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev,
170 rdev->pdev, &kfd2kgd);
e28740ec
OG		 171}
172
173void radeon_kfd_device_init(struct radeon_device *rdev)
174{
175 if (rdev->kfd) {
176 struct kgd2kfd_shared_resources gpu_resources = {
177 .compute_vmid_bitmap = 0xFF00,
178
179 .first_compute_pipe = 1,
e405ca3a 180 .compute_pipe_count = 4 - 1,
e28740ec
OG		 181 };
182
183 radeon_doorbell_get_kfd_info(rdev,
184 &gpu_resources.doorbell_physical_address,
185 &gpu_resources.doorbell_aperture_size,
186 &gpu_resources.doorbell_start_offset);
187
188 kgd2kfd->device_init(rdev->kfd, &gpu_resources);
189 }
190}
191
192void radeon_kfd_device_fini(struct radeon_device *rdev)
193{
194 if (rdev->kfd) {
195 kgd2kfd->device_exit(rdev->kfd);
196 rdev->kfd = NULL;
197 }
198}
199
200void radeon_kfd_interrupt(struct radeon_device *rdev, const void *ih_ring_entry)
201{
202 if (rdev->kfd)
203 kgd2kfd->interrupt(rdev->kfd, ih_ring_entry);
204}
205
206void radeon_kfd_suspend(struct radeon_device *rdev)
207{
208 if (rdev->kfd)
209 kgd2kfd->suspend(rdev->kfd);
210}
211
212int radeon_kfd_resume(struct radeon_device *rdev)
213{
214 int r = 0;
215
216 if (rdev->kfd)
217 r = kgd2kfd->resume(rdev->kfd);
218
219 return r;
220}
221
ceae881b
OG		 222static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
223 void **mem_obj, uint64_t *gpu_addr,
224 void **cpu_ptr)
225{
226 struct radeon_device *rdev = (struct radeon_device *)kgd;
227 struct kgd_mem **mem = (struct kgd_mem **) mem_obj;
228 int r;
229
230 BUG_ON(kgd == NULL);
231 BUG_ON(gpu_addr == NULL);
232 BUG_ON(cpu_ptr == NULL);
233
234 *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
235 if ((*mem) == NULL)
236 return -ENOMEM;
237
238 r = radeon_bo_create(rdev, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_GTT,
239 RADEON_GEM_GTT_WC, NULL, NULL, &(*mem)->bo);
240 if (r) {
241 dev_err(rdev->dev,
242 "failed to allocate BO for amdkfd (%d)\n", r);
243 return r;
244 }
245
246 /* map the buffer */
247 r = radeon_bo_reserve((*mem)->bo, true);
248 if (r) {
249 dev_err(rdev->dev, "(%d) failed to reserve bo for amdkfd\n", r);
250 goto allocate_mem_reserve_bo_failed;
251 }
252
253 r = radeon_bo_pin((*mem)->bo, RADEON_GEM_DOMAIN_GTT,
254 &(*mem)->gpu_addr);
255 if (r) {
256 dev_err(rdev->dev, "(%d) failed to pin bo for amdkfd\n", r);
257 goto allocate_mem_pin_bo_failed;
258 }
259 *gpu_addr = (*mem)->gpu_addr;
260
261 r = radeon_bo_kmap((*mem)->bo, &(*mem)->cpu_ptr);
262 if (r) {
263 dev_err(rdev->dev,
264 "(%d) failed to map bo to kernel for amdkfd\n", r);
265 goto allocate_mem_kmap_bo_failed;
266 }
267 *cpu_ptr = (*mem)->cpu_ptr;
268
269 radeon_bo_unreserve((*mem)->bo);
270
271 return 0;
272
273allocate_mem_kmap_bo_failed:
274 radeon_bo_unpin((*mem)->bo);
275allocate_mem_pin_bo_failed:
276 radeon_bo_unreserve((*mem)->bo);
277allocate_mem_reserve_bo_failed:
278 radeon_bo_unref(&(*mem)->bo);
279
280 return r;
281}
282
283static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
284{
285 struct kgd_mem *mem = (struct kgd_mem *) mem_obj;
286
287 BUG_ON(mem == NULL);
288
289 radeon_bo_reserve(mem->bo, true);
290 radeon_bo_kunmap(mem->bo);
291 radeon_bo_unpin(mem->bo);
292 radeon_bo_unreserve(mem->bo);
293 radeon_bo_unref(&(mem->bo));
294 kfree(mem);
295}
296
e28740ec
OG		 297static uint64_t get_vmem_size(struct kgd_dev *kgd)
298{
299 struct radeon_device *rdev = (struct radeon_device *)kgd;
300
301 BUG_ON(kgd == NULL);
302
303 return rdev->mc.real_vram_size;
304}
305
306static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
307{
308 struct radeon_device *rdev = (struct radeon_device *)kgd;
309
310 return rdev->asic->get_gpu_clock_counter(rdev);
311}
312
313static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
314{
315 struct radeon_device *rdev = (struct radeon_device *)kgd;
316
317 /* The sclk is in quantas of 10kHz */
318 return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
319}
320
321static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd)
322{
323 return (struct radeon_device *)kgd;
324}
325
326static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value)
327{
328 struct radeon_device *rdev = get_radeon_device(kgd);
329
330 writel(value, (void __iomem *)(rdev->rmmio + offset));
331}
332
333static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset)
334{
335 struct radeon_device *rdev = get_radeon_device(kgd);
336
337 return readl((void __iomem *)(rdev->rmmio + offset));
338}
339
340static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
341 uint32_t queue, uint32_t vmid)
342{
343 struct radeon_device *rdev = get_radeon_device(kgd);
344 uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
345
346 mutex_lock(&rdev->srbm_mutex);
347 write_register(kgd, SRBM_GFX_CNTL, value);
348}
349
350static void unlock_srbm(struct kgd_dev *kgd)
351{
352 struct radeon_device *rdev = get_radeon_device(kgd);
353
354 write_register(kgd, SRBM_GFX_CNTL, 0);
355 mutex_unlock(&rdev->srbm_mutex);
356}
357
358static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
359 uint32_t queue_id)
360{
361 uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
362 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
363
364 lock_srbm(kgd, mec, pipe, queue_id, 0);
365}
366
367static void release_queue(struct kgd_dev *kgd)
368{
369 unlock_srbm(kgd);
370}
371
372static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
373 uint32_t sh_mem_config,
374 uint32_t sh_mem_ape1_base,
375 uint32_t sh_mem_ape1_limit,
376 uint32_t sh_mem_bases)
377{
378 lock_srbm(kgd, 0, 0, 0, vmid);
379
380 write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
381 write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base);
382 write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
383 write_register(kgd, SH_MEM_BASES, sh_mem_bases);
384
385 unlock_srbm(kgd);
386}
387
388static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
389 unsigned int vmid)
390{
391 /*
392 * We have to assume that there is no outstanding mapping.
393 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0
394 * because a mapping is in progress or because a mapping finished and
395 * the SW cleared it.
396 * So the protocol is to always wait & clear.
397 */
398 uint32_t pasid_mapping = (pasid == 0) ? 0 :
399 (uint32_t)pasid | ATC_VMID_PASID_MAPPING_VALID;
400
401 write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t),
402 pasid_mapping);
403
404 while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) &
405 (1U << vmid)))
406 cpu_relax();
407 write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
408
fec77bb5
BG		 409 /* Mapping vmid to pasid also for IH block */
410 write_register(kgd, IH_VMID_0_LUT + vmid * sizeof(uint32_t),
411 pasid_mapping);
412
e28740ec
OG		 413 return 0;
414}
415
e28740ec
OG		 416static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
417 uint32_t hpd_size, uint64_t hpd_gpu_addr)
418{
5a8888a3 419 uint32_t mec = (pipe_id / CIK_PIPE_PER_MEC) + 1;
e28740ec
OG		 420 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
421
422 lock_srbm(kgd, mec, pipe, 0, 0);
423 write_register(kgd, CP_HPD_EOP_BASE_ADDR,
424 lower_32_bits(hpd_gpu_addr >> 8));
425 write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI,
426 upper_32_bits(hpd_gpu_addr >> 8));
427 write_register(kgd, CP_HPD_EOP_VMID, 0);
428 write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size);
429 unlock_srbm(kgd);
430
431 return 0;
432}
433
d36b94fc
OG		 434static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
435{
436 uint32_t mec;
437 uint32_t pipe;
438
439 mec = (pipe_id / CIK_PIPE_PER_MEC) + 1;
440 pipe = (pipe_id % CIK_PIPE_PER_MEC);
441
442 lock_srbm(kgd, mec, pipe, 0, 0);
443
444 write_register(kgd, CPC_INT_CNTL,
445 TIME_STAMP_INT_ENABLE | OPCODE_ERROR_INT_ENABLE);
446
447 unlock_srbm(kgd);
448
449 return 0;
450}
451
a84a9903
BG		 452static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
453{
454 uint32_t retval;
455
456 retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
457 m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
458
459 pr_debug("kfd: sdma base address: 0x%x\n", retval);
460
461 return retval;
462}
463
e28740ec
OG		 464static inline struct cik_mqd *get_mqd(void *mqd)
465{
466 return (struct cik_mqd *)mqd;
467}
468
a84a9903
BG		 469static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
470{
471 return (struct cik_sdma_rlc_registers *)mqd;
472}
473
e28740ec
OG		 474static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
475 uint32_t queue_id, uint32_t __user *wptr)
476{
477 uint32_t wptr_shadow, is_wptr_shadow_valid;
478 struct cik_mqd *m;
479
480 m = get_mqd(mqd);
481
482 is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
483
484 acquire_queue(kgd, pipe_id, queue_id);
485 write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);
486 write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);
487 write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control);
488
489 write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);
490 write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);
491 write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);
492
493 write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control);
494 write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo);
495 write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi);
496
497 write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr);
498
499 write_register(kgd, CP_HQD_PERSISTENT_STATE,
500 m->cp_hqd_persistent_state);
501 write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd);
502 write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type);
503
504 write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO,
505 m->cp_hqd_atomic0_preop_lo);
506
507 write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI,
508 m->cp_hqd_atomic0_preop_hi);
509
510 write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO,
511 m->cp_hqd_atomic1_preop_lo);
512
513 write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI,
514 m->cp_hqd_atomic1_preop_hi);
515
516 write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR,
517 m->cp_hqd_pq_rptr_report_addr_lo);
518
519 write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
520 m->cp_hqd_pq_rptr_report_addr_hi);
521
522 write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr);
523
524 write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR,
525 m->cp_hqd_pq_wptr_poll_addr_lo);
526
527 write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI,
528 m->cp_hqd_pq_wptr_poll_addr_hi);
529
530 write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL,
531 m->cp_hqd_pq_doorbell_control);
532
533 write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid);
534
535 write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum);
536
537 write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);
538 write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);
539
540 write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr);
541
542 if (is_wptr_shadow_valid)
543 write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow);
544
545 write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active);
546 release_queue(kgd);
547
548 return 0;
549}
550
a84a9903
BG		 551static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
552{
553 struct cik_sdma_rlc_registers *m;
554 uint32_t sdma_base_addr;
555
556 m = get_sdma_mqd(mqd);
557 sdma_base_addr = get_sdma_base_addr(m);
558
559 write_register(kgd,
560 sdma_base_addr + SDMA0_RLC0_VIRTUAL_ADDR,
561 m->sdma_rlc_virtual_addr);
562
563 write_register(kgd,
564 sdma_base_addr + SDMA0_RLC0_RB_BASE,
565 m->sdma_rlc_rb_base);
566
567 write_register(kgd,
568 sdma_base_addr + SDMA0_RLC0_RB_BASE_HI,
569 m->sdma_rlc_rb_base_hi);
570
571 write_register(kgd,
572 sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_LO,
573 m->sdma_rlc_rb_rptr_addr_lo);
574
575 write_register(kgd,
576 sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_HI,
577 m->sdma_rlc_rb_rptr_addr_hi);
578
579 write_register(kgd,
580 sdma_base_addr + SDMA0_RLC0_DOORBELL,
581 m->sdma_rlc_doorbell);
582
583 write_register(kgd,
584 sdma_base_addr + SDMA0_RLC0_RB_CNTL,
585 m->sdma_rlc_rb_cntl);
586
587 return 0;
588}
589
b64b8afc 590static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
e28740ec
OG		 591 uint32_t pipe_id, uint32_t queue_id)
592{
593 uint32_t act;
594 bool retval = false;
595 uint32_t low, high;
596
597 acquire_queue(kgd, pipe_id, queue_id);
598 act = read_register(kgd, CP_HQD_ACTIVE);
599 if (act) {
600 low = lower_32_bits(queue_address >> 8);
601 high = upper_32_bits(queue_address >> 8);
602
603 if (low == read_register(kgd, CP_HQD_PQ_BASE) &&
604 high == read_register(kgd, CP_HQD_PQ_BASE_HI))
605 retval = true;
606 }
607 release_queue(kgd);
608 return retval;
609}
610
a84a9903
BG		 611static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
612{
613 struct cik_sdma_rlc_registers *m;
614 uint32_t sdma_base_addr;
615 uint32_t sdma_rlc_rb_cntl;
616
617 m = get_sdma_mqd(mqd);
618 sdma_base_addr = get_sdma_base_addr(m);
619
620 sdma_rlc_rb_cntl = read_register(kgd,
621 sdma_base_addr + SDMA0_RLC0_RB_CNTL);
622
623 if (sdma_rlc_rb_cntl & SDMA_RB_ENABLE)
624 return true;
625
626 return false;
627}
628
e28740ec
OG		 629static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
630 unsigned int timeout, uint32_t pipe_id,
631 uint32_t queue_id)
632{
633 uint32_t temp;
634
635 acquire_queue(kgd, pipe_id, queue_id);
636 write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0);
637
638 write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type);
639
640 while (true) {
641 temp = read_register(kgd, CP_HQD_ACTIVE);
642 if (temp & 0x1)
643 break;
644 if (timeout == 0) {
645 pr_err("kfd: cp queue preemption time out (%dms)\n",
646 temp);
4c18442e 647 release_queue(kgd);
e28740ec
OG		 648 return -ETIME;
649 }
650 msleep(20);
651 timeout -= 20;
652 }
653
654 release_queue(kgd);
655 return 0;
656}
f7694323 657
a84a9903
BG		 658static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
659 unsigned int timeout)
660{
661 struct cik_sdma_rlc_registers *m;
662 uint32_t sdma_base_addr;
663 uint32_t temp;
664
665 m = get_sdma_mqd(mqd);
666 sdma_base_addr = get_sdma_base_addr(m);
667
668 temp = read_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL);
669 temp = temp & ~SDMA_RB_ENABLE;
670 write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL, temp);
671
672 while (true) {
673 temp = read_register(kgd, sdma_base_addr +
674 SDMA0_RLC0_CONTEXT_STATUS);
675 if (temp & SDMA_RLC_IDLE)
676 break;
677 if (timeout == 0)
678 return -ETIME;
679 msleep(20);
680 timeout -= 20;
681 }
682
683 write_register(kgd, sdma_base_addr + SDMA0_RLC0_DOORBELL, 0);
684 write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_RPTR, 0);
685 write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_WPTR, 0);
686 write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_BASE, 0);
687
688 return 0;
689}
690
a6186f4d
YS		 691static int kgd_address_watch_disable(struct kgd_dev *kgd)
692{
693 union TCP_WATCH_CNTL_BITS cntl;
694 unsigned int i;
695
696 cntl.u32All = 0;
697
698 cntl.bitfields.valid = 0;
699 cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK;
700 cntl.bitfields.atc = 1;
701
702 /* Turning off this address until we set all the registers */
703 for (i = 0; i < MAX_WATCH_ADDRESSES; i++)
704 write_register(kgd,
705 watchRegs[i * ADDRESS_WATCH_REG_MAX +
706 ADDRESS_WATCH_REG_CNTL],
707 cntl.u32All);
708
709 return 0;
710}
711
712static int kgd_address_watch_execute(struct kgd_dev *kgd,
713 unsigned int watch_point_id,
714 uint32_t cntl_val,
715 uint32_t addr_hi,
716 uint32_t addr_lo)
717{
718 union TCP_WATCH_CNTL_BITS cntl;
719
720 cntl.u32All = cntl_val;
721
722 /* Turning off this watch point until we set all the registers */
723 cntl.bitfields.valid = 0;
724 write_register(kgd,
725 watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
726 ADDRESS_WATCH_REG_CNTL],
727 cntl.u32All);
728
729 write_register(kgd,
730 watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
731 ADDRESS_WATCH_REG_ADDR_HI],
732 addr_hi);
733
734 write_register(kgd,
735 watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
736 ADDRESS_WATCH_REG_ADDR_LO],
737 addr_lo);
738
739 /* Enable the watch point */
740 cntl.bitfields.valid = 1;
741
742 write_register(kgd,
743 watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
744 ADDRESS_WATCH_REG_CNTL],
745 cntl.u32All);
746
747 return 0;
748}
749
750static int kgd_wave_control_execute(struct kgd_dev *kgd,
751 uint32_t gfx_index_val,
752 uint32_t sq_cmd)
753{
754 struct radeon_device *rdev = get_radeon_device(kgd);
755 uint32_t data;
756
757 mutex_lock(&rdev->grbm_idx_mutex);
758
759 write_register(kgd, GRBM_GFX_INDEX, gfx_index_val);
760 write_register(kgd, SQ_CMD, sq_cmd);
761
762 /* Restore the GRBM_GFX_INDEX register */
763
764 data = INSTANCE_BROADCAST_WRITES | SH_BROADCAST_WRITES |
765 SE_BROADCAST_WRITES;
766
767 write_register(kgd, GRBM_GFX_INDEX, data);
768
769 mutex_unlock(&rdev->grbm_idx_mutex);
770
771 return 0;
772}
773
774static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
775 unsigned int watch_point_id,
776 unsigned int reg_offset)
777{
778 return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset];
779}
780
f7694323
OG		 781static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
782{
783 struct radeon_device *rdev = (struct radeon_device *) kgd;
784 const union radeon_firmware_header *hdr;
785
786 BUG_ON(kgd == NULL || rdev->mec_fw == NULL);
787
788 switch (type) {
789 case KGD_ENGINE_PFP:
790 hdr = (const union radeon_firmware_header *) rdev->pfp_fw->data;
791 break;
792
793 case KGD_ENGINE_ME:
794 hdr = (const union radeon_firmware_header *) rdev->me_fw->data;
795 break;
796
797 case KGD_ENGINE_CE:
798 hdr = (const union radeon_firmware_header *) rdev->ce_fw->data;
799 break;
800
801 case KGD_ENGINE_MEC1:
802 hdr = (const union radeon_firmware_header *) rdev->mec_fw->data;
803 break;
804
805 case KGD_ENGINE_MEC2:
806 hdr = (const union radeon_firmware_header *)
807 rdev->mec2_fw->data;
808 break;
809
810 case KGD_ENGINE_RLC:
811 hdr = (const union radeon_firmware_header *) rdev->rlc_fw->data;
812 break;
813
814 case KGD_ENGINE_SDMA:
815 hdr = (const union radeon_firmware_header *)
816 rdev->sdma_fw->data;
817 break;
818
819 default:
820 return 0;
821 }
822
823 if (hdr == NULL)
824 return 0;
825
826 /* Only 12 bit in use*/
827 return hdr->common.ucode_version;
828}
Linux kernel - Deliverables
This page took 0.145786 seconds and 5 git commands to generate.