2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/device.h>
24 #include <linux/export.h>
25 #include <linux/err.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
29 #include <linux/uaccess.h>
30 #include <linux/compat.h>
31 #include <uapi/linux/kfd_ioctl.h>
32 #include <linux/time.h>
34 #include <linux/mman.h>
35 #include <asm/processor.h>
37 #include "kfd_device_queue_manager.h"
38 #include "kfd_dbgmgr.h"
40 static long kfd_ioctl(struct file
*, unsigned int, unsigned long);
41 static int kfd_open(struct inode
*, struct file
*);
42 static int kfd_mmap(struct file
*, struct vm_area_struct
*);
44 static const char kfd_dev_name
[] = "kfd";
46 static const struct file_operations kfd_fops
= {
48 .unlocked_ioctl
= kfd_ioctl
,
49 .compat_ioctl
= kfd_ioctl
,
54 static int kfd_char_dev_major
= -1;
55 static struct class *kfd_class
;
56 struct device
*kfd_device
;
58 int kfd_chardev_init(void)
62 kfd_char_dev_major
= register_chrdev(0, kfd_dev_name
, &kfd_fops
);
63 err
= kfd_char_dev_major
;
65 goto err_register_chrdev
;
67 kfd_class
= class_create(THIS_MODULE
, kfd_dev_name
);
68 err
= PTR_ERR(kfd_class
);
69 if (IS_ERR(kfd_class
))
70 goto err_class_create
;
72 kfd_device
= device_create(kfd_class
, NULL
,
73 MKDEV(kfd_char_dev_major
, 0),
75 err
= PTR_ERR(kfd_device
);
76 if (IS_ERR(kfd_device
))
77 goto err_device_create
;
82 class_destroy(kfd_class
);
84 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
89 void kfd_chardev_exit(void)
91 device_destroy(kfd_class
, MKDEV(kfd_char_dev_major
, 0));
92 class_destroy(kfd_class
);
93 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
96 struct device
*kfd_chardev(void)
102 static int kfd_open(struct inode
*inode
, struct file
*filep
)
104 struct kfd_process
*process
;
105 bool is_32bit_user_mode
;
107 if (iminor(inode
) != 0)
110 is_32bit_user_mode
= is_compat_task();
112 if (is_32bit_user_mode
== true) {
114 "Process %d (32-bit) failed to open /dev/kfd\n"
115 "32-bit processes are not supported by amdkfd\n",
120 process
= kfd_create_process(current
);
122 return PTR_ERR(process
);
124 dev_dbg(kfd_device
, "process %d opened, compat mode (32 bit) - %d\n",
125 process
->pasid
, process
->is_32bit_user_mode
);
130 static int kfd_ioctl_get_version(struct file
*filep
, struct kfd_process
*p
,
133 struct kfd_ioctl_get_version_args
*args
= data
;
136 args
->major_version
= KFD_IOCTL_MAJOR_VERSION
;
137 args
->minor_version
= KFD_IOCTL_MINOR_VERSION
;
142 static int set_queue_properties_from_user(struct queue_properties
*q_properties
,
143 struct kfd_ioctl_create_queue_args
*args
)
145 if (args
->queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
146 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
150 if (args
->queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
151 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
155 if ((args
->ring_base_address
) &&
156 (!access_ok(VERIFY_WRITE
,
157 (const void __user
*) args
->ring_base_address
,
158 sizeof(uint64_t)))) {
159 pr_err("kfd: can't access ring base address\n");
163 if (!is_power_of_2(args
->ring_size
) && (args
->ring_size
!= 0)) {
164 pr_err("kfd: ring size must be a power of 2 or 0\n");
168 if (!access_ok(VERIFY_WRITE
,
169 (const void __user
*) args
->read_pointer_address
,
171 pr_err("kfd: can't access read pointer\n");
175 if (!access_ok(VERIFY_WRITE
,
176 (const void __user
*) args
->write_pointer_address
,
178 pr_err("kfd: can't access write pointer\n");
182 if (args
->eop_buffer_address
&&
183 !access_ok(VERIFY_WRITE
,
184 (const void __user
*) args
->eop_buffer_address
,
186 pr_debug("kfd: can't access eop buffer");
190 if (args
->ctx_save_restore_address
&&
191 !access_ok(VERIFY_WRITE
,
192 (const void __user
*) args
->ctx_save_restore_address
,
194 pr_debug("kfd: can't access ctx save restore buffer");
198 q_properties
->is_interop
= false;
199 q_properties
->queue_percent
= args
->queue_percentage
;
200 q_properties
->priority
= args
->queue_priority
;
201 q_properties
->queue_address
= args
->ring_base_address
;
202 q_properties
->queue_size
= args
->ring_size
;
203 q_properties
->read_ptr
= (uint32_t *) args
->read_pointer_address
;
204 q_properties
->write_ptr
= (uint32_t *) args
->write_pointer_address
;
205 q_properties
->eop_ring_buffer_address
= args
->eop_buffer_address
;
206 q_properties
->eop_ring_buffer_size
= args
->eop_buffer_size
;
207 q_properties
->ctx_save_restore_area_address
=
208 args
->ctx_save_restore_address
;
209 q_properties
->ctx_save_restore_area_size
= args
->ctx_save_restore_size
;
210 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE
||
211 args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
212 q_properties
->type
= KFD_QUEUE_TYPE_COMPUTE
;
213 else if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_SDMA
)
214 q_properties
->type
= KFD_QUEUE_TYPE_SDMA
;
218 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
219 q_properties
->format
= KFD_QUEUE_FORMAT_AQL
;
221 q_properties
->format
= KFD_QUEUE_FORMAT_PM4
;
223 pr_debug("Queue Percentage (%d, %d)\n",
224 q_properties
->queue_percent
, args
->queue_percentage
);
226 pr_debug("Queue Priority (%d, %d)\n",
227 q_properties
->priority
, args
->queue_priority
);
229 pr_debug("Queue Address (0x%llX, 0x%llX)\n",
230 q_properties
->queue_address
, args
->ring_base_address
);
232 pr_debug("Queue Size (0x%llX, %u)\n",
233 q_properties
->queue_size
, args
->ring_size
);
235 pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
236 (uint64_t) q_properties
->read_ptr
,
237 (uint64_t) q_properties
->write_ptr
);
239 pr_debug("Queue Format (%d)\n", q_properties
->format
);
241 pr_debug("Queue EOP (0x%llX)\n", q_properties
->eop_ring_buffer_address
);
243 pr_debug("Queue CTX save arex (0x%llX)\n",
244 q_properties
->ctx_save_restore_area_address
);
249 static int kfd_ioctl_create_queue(struct file
*filep
, struct kfd_process
*p
,
252 struct kfd_ioctl_create_queue_args
*args
= data
;
255 unsigned int queue_id
;
256 struct kfd_process_device
*pdd
;
257 struct queue_properties q_properties
;
259 memset(&q_properties
, 0, sizeof(struct queue_properties
));
261 pr_debug("kfd: creating queue ioctl\n");
263 err
= set_queue_properties_from_user(&q_properties
, args
);
267 pr_debug("kfd: looking for gpu id 0x%x\n", args
->gpu_id
);
268 dev
= kfd_device_by_id(args
->gpu_id
);
270 pr_debug("kfd: gpu id 0x%x was not found\n", args
->gpu_id
);
274 mutex_lock(&p
->mutex
);
276 pdd
= kfd_bind_process_to_device(dev
, p
);
279 goto err_bind_process
;
282 pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
286 err
= pqm_create_queue(&p
->pqm
, dev
, filep
, &q_properties
,
287 0, q_properties
.type
, &queue_id
);
289 goto err_create_queue
;
291 args
->queue_id
= queue_id
;
294 /* Return gpu_id as doorbell offset for mmap usage */
295 args
->doorbell_offset
= (KFD_MMAP_DOORBELL_MASK
| args
->gpu_id
);
296 args
->doorbell_offset
<<= PAGE_SHIFT
;
298 mutex_unlock(&p
->mutex
);
300 pr_debug("kfd: queue id %d was created successfully\n", args
->queue_id
);
302 pr_debug("ring buffer address == 0x%016llX\n",
303 args
->ring_base_address
);
305 pr_debug("read ptr address == 0x%016llX\n",
306 args
->read_pointer_address
);
308 pr_debug("write ptr address == 0x%016llX\n",
309 args
->write_pointer_address
);
315 mutex_unlock(&p
->mutex
);
319 static int kfd_ioctl_destroy_queue(struct file
*filp
, struct kfd_process
*p
,
323 struct kfd_ioctl_destroy_queue_args
*args
= data
;
325 pr_debug("kfd: destroying queue id %d for PASID %d\n",
329 mutex_lock(&p
->mutex
);
331 retval
= pqm_destroy_queue(&p
->pqm
, args
->queue_id
);
333 mutex_unlock(&p
->mutex
);
337 static int kfd_ioctl_update_queue(struct file
*filp
, struct kfd_process
*p
,
341 struct kfd_ioctl_update_queue_args
*args
= data
;
342 struct queue_properties properties
;
344 if (args
->queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
345 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
349 if (args
->queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
350 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
354 if ((args
->ring_base_address
) &&
355 (!access_ok(VERIFY_WRITE
,
356 (const void __user
*) args
->ring_base_address
,
357 sizeof(uint64_t)))) {
358 pr_err("kfd: can't access ring base address\n");
362 if (!is_power_of_2(args
->ring_size
) && (args
->ring_size
!= 0)) {
363 pr_err("kfd: ring size must be a power of 2 or 0\n");
367 properties
.queue_address
= args
->ring_base_address
;
368 properties
.queue_size
= args
->ring_size
;
369 properties
.queue_percent
= args
->queue_percentage
;
370 properties
.priority
= args
->queue_priority
;
372 pr_debug("kfd: updating queue id %d for PASID %d\n",
373 args
->queue_id
, p
->pasid
);
375 mutex_lock(&p
->mutex
);
377 retval
= pqm_update_queue(&p
->pqm
, args
->queue_id
, &properties
);
379 mutex_unlock(&p
->mutex
);
384 static int kfd_ioctl_set_memory_policy(struct file
*filep
,
385 struct kfd_process
*p
, void *data
)
387 struct kfd_ioctl_set_memory_policy_args
*args
= data
;
390 struct kfd_process_device
*pdd
;
391 enum cache_policy default_policy
, alternate_policy
;
393 if (args
->default_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
394 && args
->default_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
398 if (args
->alternate_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
399 && args
->alternate_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
403 dev
= kfd_device_by_id(args
->gpu_id
);
407 mutex_lock(&p
->mutex
);
409 pdd
= kfd_bind_process_to_device(dev
, p
);
415 default_policy
= (args
->default_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
416 ? cache_policy_coherent
: cache_policy_noncoherent
;
419 (args
->alternate_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
420 ? cache_policy_coherent
: cache_policy_noncoherent
;
422 if (!dev
->dqm
->ops
.set_cache_memory_policy(dev
->dqm
,
426 (void __user
*)args
->alternate_aperture_base
,
427 args
->alternate_aperture_size
))
431 mutex_unlock(&p
->mutex
);
436 static int kfd_ioctl_dbg_register(struct file
*filep
,
437 struct kfd_process
*p
, void *data
)
439 struct kfd_ioctl_dbg_register_args
*args
= data
;
441 struct kfd_dbgmgr
*dbgmgr_ptr
;
442 struct kfd_process_device
*pdd
;
446 dev
= kfd_device_by_id(args
->gpu_id
);
450 if (dev
->device_info
->asic_family
== CHIP_CARRIZO
) {
451 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
455 mutex_lock(kfd_get_dbgmgr_mutex());
456 mutex_lock(&p
->mutex
);
459 * make sure that we have pdd, if this the first queue created for
462 pdd
= kfd_bind_process_to_device(dev
, p
);
464 mutex_unlock(&p
->mutex
);
465 mutex_unlock(kfd_get_dbgmgr_mutex());
469 if (dev
->dbgmgr
== NULL
) {
470 /* In case of a legal call, we have no dbgmgr yet */
471 create_ok
= kfd_dbgmgr_create(&dbgmgr_ptr
, dev
);
473 status
= kfd_dbgmgr_register(dbgmgr_ptr
, p
);
475 kfd_dbgmgr_destroy(dbgmgr_ptr
);
477 dev
->dbgmgr
= dbgmgr_ptr
;
480 pr_debug("debugger already registered\n");
484 mutex_unlock(&p
->mutex
);
485 mutex_unlock(kfd_get_dbgmgr_mutex());
490 static int kfd_ioctl_dbg_unrgesiter(struct file
*filep
,
491 struct kfd_process
*p
, void *data
)
493 struct kfd_ioctl_dbg_unregister_args
*args
= data
;
497 dev
= kfd_device_by_id(args
->gpu_id
);
501 if (dev
->device_info
->asic_family
== CHIP_CARRIZO
) {
502 pr_debug("kfd_ioctl_dbg_unrgesiter not supported on CZ\n");
506 mutex_lock(kfd_get_dbgmgr_mutex());
508 status
= kfd_dbgmgr_unregister(dev
->dbgmgr
, p
);
510 kfd_dbgmgr_destroy(dev
->dbgmgr
);
514 mutex_unlock(kfd_get_dbgmgr_mutex());
520 * Parse and generate variable size data structure for address watch.
521 * Total size of the buffer and # watch points is limited in order
522 * to prevent kernel abuse. (no bearing to the much smaller HW limitation
523 * which is enforced by dbgdev module)
524 * please also note that the watch address itself are not "copied from user",
525 * since it be set into the HW in user mode values.
528 static int kfd_ioctl_dbg_address_watch(struct file
*filep
,
529 struct kfd_process
*p
, void *data
)
531 struct kfd_ioctl_dbg_address_watch_args
*args
= data
;
533 struct dbg_address_watch_info aw_info
;
534 unsigned char *args_buff
;
536 void __user
*cmd_from_user
;
537 uint64_t watch_mask_value
= 0;
538 unsigned int args_idx
= 0;
540 memset((void *) &aw_info
, 0, sizeof(struct dbg_address_watch_info
));
542 dev
= kfd_device_by_id(args
->gpu_id
);
546 if (dev
->device_info
->asic_family
== CHIP_CARRIZO
) {
547 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
551 cmd_from_user
= (void __user
*) args
->content_ptr
;
553 /* Validate arguments */
555 if ((args
->buf_size_in_bytes
> MAX_ALLOWED_AW_BUFF_SIZE
) ||
556 (args
->buf_size_in_bytes
<= sizeof(*args
) + sizeof(int) * 2) ||
557 (cmd_from_user
== NULL
))
560 /* this is the actual buffer to work with */
562 args_buff
= memdup_user(args_buff
,
563 args
->buf_size_in_bytes
- sizeof(*args
));
564 if (IS_ERR(args_buff
))
565 return PTR_ERR(args_buff
);
569 aw_info
.num_watch_points
= *((uint32_t *)(&args_buff
[args_idx
]));
570 args_idx
+= sizeof(aw_info
.num_watch_points
);
572 aw_info
.watch_mode
= (enum HSA_DBG_WATCH_MODE
*) &args_buff
[args_idx
];
573 args_idx
+= sizeof(enum HSA_DBG_WATCH_MODE
) * aw_info
.num_watch_points
;
576 * set watch address base pointer to point on the array base
579 aw_info
.watch_address
= (uint64_t *) &args_buff
[args_idx
];
581 /* skip over the addresses buffer */
582 args_idx
+= sizeof(aw_info
.watch_address
) * aw_info
.num_watch_points
;
584 if (args_idx
>= args
->buf_size_in_bytes
- sizeof(*args
)) {
589 watch_mask_value
= (uint64_t) args_buff
[args_idx
];
591 if (watch_mask_value
> 0) {
593 * There is an array of masks.
594 * set watch mask base pointer to point on the array base
597 aw_info
.watch_mask
= (uint64_t *) &args_buff
[args_idx
];
599 /* skip over the masks buffer */
600 args_idx
+= sizeof(aw_info
.watch_mask
) *
601 aw_info
.num_watch_points
;
603 /* just the NULL mask, set to NULL and skip over it */
604 aw_info
.watch_mask
= NULL
;
605 args_idx
+= sizeof(aw_info
.watch_mask
);
608 if (args_idx
>= args
->buf_size_in_bytes
- sizeof(args
)) {
613 /* Currently HSA Event is not supported for DBG */
614 aw_info
.watch_event
= NULL
;
616 mutex_lock(kfd_get_dbgmgr_mutex());
618 status
= kfd_dbgmgr_address_watch(dev
->dbgmgr
, &aw_info
);
620 mutex_unlock(kfd_get_dbgmgr_mutex());
627 /* Parse and generate fixed size data structure for wave control */
628 static int kfd_ioctl_dbg_wave_control(struct file
*filep
,
629 struct kfd_process
*p
, void *data
)
631 struct kfd_ioctl_dbg_wave_control_args
*args
= data
;
633 struct dbg_wave_control_info wac_info
;
634 unsigned char *args_buff
;
635 uint32_t computed_buff_size
;
637 void __user
*cmd_from_user
;
638 unsigned int args_idx
= 0;
640 memset((void *) &wac_info
, 0, sizeof(struct dbg_wave_control_info
));
642 /* we use compact form, independent of the packing attribute value */
643 computed_buff_size
= sizeof(*args
) +
644 sizeof(wac_info
.mode
) +
645 sizeof(wac_info
.operand
) +
646 sizeof(wac_info
.dbgWave_msg
.DbgWaveMsg
) +
647 sizeof(wac_info
.dbgWave_msg
.MemoryVA
) +
648 sizeof(wac_info
.trapId
);
650 dev
= kfd_device_by_id(args
->gpu_id
);
654 if (dev
->device_info
->asic_family
== CHIP_CARRIZO
) {
655 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
659 /* input size must match the computed "compact" size */
660 if (args
->buf_size_in_bytes
!= computed_buff_size
) {
661 pr_debug("size mismatch, computed : actual %u : %u\n",
662 args
->buf_size_in_bytes
, computed_buff_size
);
666 cmd_from_user
= (void __user
*) args
->content_ptr
;
668 if (cmd_from_user
== NULL
)
671 /* copy the entire buffer from user */
673 args_buff
= memdup_user(cmd_from_user
,
674 args
->buf_size_in_bytes
- sizeof(*args
));
675 if (IS_ERR(args_buff
))
676 return PTR_ERR(args_buff
);
678 /* move ptr to the start of the "pay-load" area */
679 wac_info
.process
= p
;
681 wac_info
.operand
= *((enum HSA_DBG_WAVEOP
*)(&args_buff
[args_idx
]));
682 args_idx
+= sizeof(wac_info
.operand
);
684 wac_info
.mode
= *((enum HSA_DBG_WAVEMODE
*)(&args_buff
[args_idx
]));
685 args_idx
+= sizeof(wac_info
.mode
);
687 wac_info
.trapId
= *((uint32_t *)(&args_buff
[args_idx
]));
688 args_idx
+= sizeof(wac_info
.trapId
);
690 wac_info
.dbgWave_msg
.DbgWaveMsg
.WaveMsgInfoGen2
.Value
=
691 *((uint32_t *)(&args_buff
[args_idx
]));
692 wac_info
.dbgWave_msg
.MemoryVA
= NULL
;
694 mutex_lock(kfd_get_dbgmgr_mutex());
696 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n",
697 wac_info
.process
, wac_info
.operand
,
698 wac_info
.mode
, wac_info
.trapId
,
699 wac_info
.dbgWave_msg
.DbgWaveMsg
.WaveMsgInfoGen2
.Value
);
701 status
= kfd_dbgmgr_wave_control(dev
->dbgmgr
, &wac_info
);
703 pr_debug("Returned status of dbg manager is %ld\n", status
);
705 mutex_unlock(kfd_get_dbgmgr_mutex());
712 static int kfd_ioctl_get_clock_counters(struct file
*filep
,
713 struct kfd_process
*p
, void *data
)
715 struct kfd_ioctl_get_clock_counters_args
*args
= data
;
717 struct timespec64 time
;
719 dev
= kfd_device_by_id(args
->gpu_id
);
723 /* Reading GPU clock counter from KGD */
724 args
->gpu_clock_counter
=
725 dev
->kfd2kgd
->get_gpu_clock_counter(dev
->kgd
);
727 /* No access to rdtsc. Using raw monotonic time */
728 getrawmonotonic64(&time
);
729 args
->cpu_clock_counter
= (uint64_t)timespec64_to_ns(&time
);
731 get_monotonic_boottime64(&time
);
732 args
->system_clock_counter
= (uint64_t)timespec64_to_ns(&time
);
734 /* Since the counter is in nano-seconds we use 1GHz frequency */
735 args
->system_clock_freq
= 1000000000;
741 static int kfd_ioctl_get_process_apertures(struct file
*filp
,
742 struct kfd_process
*p
, void *data
)
744 struct kfd_ioctl_get_process_apertures_args
*args
= data
;
745 struct kfd_process_device_apertures
*pAperture
;
746 struct kfd_process_device
*pdd
;
748 dev_dbg(kfd_device
, "get apertures for PASID %d", p
->pasid
);
750 args
->num_of_nodes
= 0;
752 mutex_lock(&p
->mutex
);
754 /*if the process-device list isn't empty*/
755 if (kfd_has_process_device_data(p
)) {
756 /* Run over all pdd of the process */
757 pdd
= kfd_get_first_process_device_data(p
);
760 &args
->process_apertures
[args
->num_of_nodes
];
761 pAperture
->gpu_id
= pdd
->dev
->id
;
762 pAperture
->lds_base
= pdd
->lds_base
;
763 pAperture
->lds_limit
= pdd
->lds_limit
;
764 pAperture
->gpuvm_base
= pdd
->gpuvm_base
;
765 pAperture
->gpuvm_limit
= pdd
->gpuvm_limit
;
766 pAperture
->scratch_base
= pdd
->scratch_base
;
767 pAperture
->scratch_limit
= pdd
->scratch_limit
;
770 "node id %u\n", args
->num_of_nodes
);
772 "gpu id %u\n", pdd
->dev
->id
);
774 "lds_base %llX\n", pdd
->lds_base
);
776 "lds_limit %llX\n", pdd
->lds_limit
);
778 "gpuvm_base %llX\n", pdd
->gpuvm_base
);
780 "gpuvm_limit %llX\n", pdd
->gpuvm_limit
);
782 "scratch_base %llX\n", pdd
->scratch_base
);
784 "scratch_limit %llX\n", pdd
->scratch_limit
);
786 args
->num_of_nodes
++;
787 } while ((pdd
= kfd_get_next_process_device_data(p
, pdd
)) != NULL
&&
788 (args
->num_of_nodes
< NUM_OF_SUPPORTED_GPUS
));
791 mutex_unlock(&p
->mutex
);
796 static int kfd_ioctl_create_event(struct file
*filp
, struct kfd_process
*p
,
799 struct kfd_ioctl_create_event_args
*args
= data
;
802 err
= kfd_event_create(filp
, p
, args
->event_type
,
803 args
->auto_reset
!= 0, args
->node_id
,
804 &args
->event_id
, &args
->event_trigger_data
,
805 &args
->event_page_offset
,
806 &args
->event_slot_index
);
811 static int kfd_ioctl_destroy_event(struct file
*filp
, struct kfd_process
*p
,
814 struct kfd_ioctl_destroy_event_args
*args
= data
;
816 return kfd_event_destroy(p
, args
->event_id
);
819 static int kfd_ioctl_set_event(struct file
*filp
, struct kfd_process
*p
,
822 struct kfd_ioctl_set_event_args
*args
= data
;
824 return kfd_set_event(p
, args
->event_id
);
827 static int kfd_ioctl_reset_event(struct file
*filp
, struct kfd_process
*p
,
830 struct kfd_ioctl_reset_event_args
*args
= data
;
832 return kfd_reset_event(p
, args
->event_id
);
835 static int kfd_ioctl_wait_events(struct file
*filp
, struct kfd_process
*p
,
838 struct kfd_ioctl_wait_events_args
*args
= data
;
839 enum kfd_event_wait_result wait_result
;
842 err
= kfd_wait_on_events(p
, args
->num_events
,
843 (void __user
*)args
->events_ptr
,
844 (args
->wait_for_all
!= 0),
845 args
->timeout
, &wait_result
);
847 args
->wait_result
= wait_result
;
852 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
853 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl}
856 static const struct amdkfd_ioctl_desc amdkfd_ioctls
[] = {
857 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION
,
858 kfd_ioctl_get_version
, 0),
860 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE
,
861 kfd_ioctl_create_queue
, 0),
863 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE
,
864 kfd_ioctl_destroy_queue
, 0),
866 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY
,
867 kfd_ioctl_set_memory_policy
, 0),
869 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS
,
870 kfd_ioctl_get_clock_counters
, 0),
872 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES
,
873 kfd_ioctl_get_process_apertures
, 0),
875 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE
,
876 kfd_ioctl_update_queue
, 0),
878 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT
,
879 kfd_ioctl_create_event
, 0),
881 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT
,
882 kfd_ioctl_destroy_event
, 0),
884 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT
,
885 kfd_ioctl_set_event
, 0),
887 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT
,
888 kfd_ioctl_reset_event
, 0),
890 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS
,
891 kfd_ioctl_wait_events
, 0),
893 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER
,
894 kfd_ioctl_dbg_register
, 0),
896 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER
,
897 kfd_ioctl_dbg_unrgesiter
, 0),
899 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH
,
900 kfd_ioctl_dbg_address_watch
, 0),
902 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL
,
903 kfd_ioctl_dbg_wave_control
, 0),
906 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
908 static long kfd_ioctl(struct file
*filep
, unsigned int cmd
, unsigned long arg
)
910 struct kfd_process
*process
;
911 amdkfd_ioctl_t
*func
;
912 const struct amdkfd_ioctl_desc
*ioctl
= NULL
;
913 unsigned int nr
= _IOC_NR(cmd
);
914 char stack_kdata
[128];
916 unsigned int usize
, asize
;
917 int retcode
= -EINVAL
;
919 if (nr
>= AMDKFD_CORE_IOCTL_COUNT
)
922 if ((nr
>= AMDKFD_COMMAND_START
) && (nr
< AMDKFD_COMMAND_END
)) {
925 ioctl
= &amdkfd_ioctls
[nr
];
927 amdkfd_size
= _IOC_SIZE(ioctl
->cmd
);
928 usize
= asize
= _IOC_SIZE(cmd
);
929 if (amdkfd_size
> asize
)
936 dev_dbg(kfd_device
, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd
, nr
, arg
);
938 process
= kfd_get_process(current
);
939 if (IS_ERR(process
)) {
940 dev_dbg(kfd_device
, "no process\n");
944 /* Do not trust userspace, use our own definition */
947 if (unlikely(!func
)) {
948 dev_dbg(kfd_device
, "no function\n");
953 if (cmd
& (IOC_IN
| IOC_OUT
)) {
954 if (asize
<= sizeof(stack_kdata
)) {
957 kdata
= kmalloc(asize
, GFP_KERNEL
);
964 memset(kdata
+ usize
, 0, asize
- usize
);
968 if (copy_from_user(kdata
, (void __user
*)arg
, usize
) != 0) {
972 } else if (cmd
& IOC_OUT
) {
973 memset(kdata
, 0, usize
);
976 retcode
= func(filep
, process
, kdata
);
979 if (copy_to_user((void __user
*)arg
, kdata
, usize
) != 0)
984 dev_dbg(kfd_device
, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
985 task_pid_nr(current
), cmd
, nr
);
987 if (kdata
!= stack_kdata
)
991 dev_dbg(kfd_device
, "ret = %d\n", retcode
);
996 static int kfd_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
998 struct kfd_process
*process
;
1000 process
= kfd_get_process(current
);
1001 if (IS_ERR(process
))
1002 return PTR_ERR(process
);
1004 if ((vma
->vm_pgoff
& KFD_MMAP_DOORBELL_MASK
) ==
1005 KFD_MMAP_DOORBELL_MASK
) {
1006 vma
->vm_pgoff
= vma
->vm_pgoff
^ KFD_MMAP_DOORBELL_MASK
;
1007 return kfd_doorbell_mmap(process
, vma
);
1008 } else if ((vma
->vm_pgoff
& KFD_MMAP_EVENTS_MASK
) ==
1009 KFD_MMAP_EVENTS_MASK
) {
1010 vma
->vm_pgoff
= vma
->vm_pgoff
^ KFD_MMAP_EVENTS_MASK
;
1011 return kfd_event_mmap(process
, vma
);