2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/device.h>
24 #include <linux/export.h>
25 #include <linux/err.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
29 #include <linux/uaccess.h>
30 #include <linux/compat.h>
31 #include <uapi/linux/kfd_ioctl.h>
32 #include <linux/time.h>
34 #include <uapi/asm-generic/mman-common.h>
35 #include <asm/processor.h>
37 #include "kfd_device_queue_manager.h"
39 static long kfd_ioctl(struct file
*, unsigned int, unsigned long);
40 static int kfd_open(struct inode
*, struct file
*);
41 static int kfd_mmap(struct file
*, struct vm_area_struct
*);
43 static const char kfd_dev_name
[] = "kfd";
45 static const struct file_operations kfd_fops
= {
47 .unlocked_ioctl
= kfd_ioctl
,
48 .compat_ioctl
= kfd_ioctl
,
53 static int kfd_char_dev_major
= -1;
54 static struct class *kfd_class
;
55 struct device
*kfd_device
;
57 int kfd_chardev_init(void)
61 kfd_char_dev_major
= register_chrdev(0, kfd_dev_name
, &kfd_fops
);
62 err
= kfd_char_dev_major
;
64 goto err_register_chrdev
;
66 kfd_class
= class_create(THIS_MODULE
, kfd_dev_name
);
67 err
= PTR_ERR(kfd_class
);
68 if (IS_ERR(kfd_class
))
69 goto err_class_create
;
71 kfd_device
= device_create(kfd_class
, NULL
,
72 MKDEV(kfd_char_dev_major
, 0),
74 err
= PTR_ERR(kfd_device
);
75 if (IS_ERR(kfd_device
))
76 goto err_device_create
;
81 class_destroy(kfd_class
);
83 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
88 void kfd_chardev_exit(void)
90 device_destroy(kfd_class
, MKDEV(kfd_char_dev_major
, 0));
91 class_destroy(kfd_class
);
92 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
95 struct device
*kfd_chardev(void)
101 static int kfd_open(struct inode
*inode
, struct file
*filep
)
103 struct kfd_process
*process
;
104 bool is_32bit_user_mode
;
106 if (iminor(inode
) != 0)
109 is_32bit_user_mode
= is_compat_task();
111 if (is_32bit_user_mode
== true) {
113 "Process %d (32-bit) failed to open /dev/kfd\n"
114 "32-bit processes are not supported by amdkfd\n",
119 process
= kfd_create_process(current
);
121 return PTR_ERR(process
);
123 dev_dbg(kfd_device
, "process %d opened, compat mode (32 bit) - %d\n",
124 process
->pasid
, process
->is_32bit_user_mode
);
129 static int kfd_ioctl_get_version(struct file
*filep
, struct kfd_process
*p
,
132 struct kfd_ioctl_get_version_args
*args
= data
;
135 args
->major_version
= KFD_IOCTL_MAJOR_VERSION
;
136 args
->minor_version
= KFD_IOCTL_MINOR_VERSION
;
141 static int set_queue_properties_from_user(struct queue_properties
*q_properties
,
142 struct kfd_ioctl_create_queue_args
*args
)
144 if (args
->queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
145 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
149 if (args
->queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
150 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
154 if ((args
->ring_base_address
) &&
155 (!access_ok(VERIFY_WRITE
,
156 (const void __user
*) args
->ring_base_address
,
157 sizeof(uint64_t)))) {
158 pr_err("kfd: can't access ring base address\n");
162 if (!is_power_of_2(args
->ring_size
) && (args
->ring_size
!= 0)) {
163 pr_err("kfd: ring size must be a power of 2 or 0\n");
167 if (!access_ok(VERIFY_WRITE
,
168 (const void __user
*) args
->read_pointer_address
,
170 pr_err("kfd: can't access read pointer\n");
174 if (!access_ok(VERIFY_WRITE
,
175 (const void __user
*) args
->write_pointer_address
,
177 pr_err("kfd: can't access write pointer\n");
181 q_properties
->is_interop
= false;
182 q_properties
->queue_percent
= args
->queue_percentage
;
183 q_properties
->priority
= args
->queue_priority
;
184 q_properties
->queue_address
= args
->ring_base_address
;
185 q_properties
->queue_size
= args
->ring_size
;
186 q_properties
->read_ptr
= (uint32_t *) args
->read_pointer_address
;
187 q_properties
->write_ptr
= (uint32_t *) args
->write_pointer_address
;
188 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE
||
189 args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
190 q_properties
->type
= KFD_QUEUE_TYPE_COMPUTE
;
194 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
195 q_properties
->format
= KFD_QUEUE_FORMAT_AQL
;
197 q_properties
->format
= KFD_QUEUE_FORMAT_PM4
;
199 pr_debug("Queue Percentage (%d, %d)\n",
200 q_properties
->queue_percent
, args
->queue_percentage
);
202 pr_debug("Queue Priority (%d, %d)\n",
203 q_properties
->priority
, args
->queue_priority
);
205 pr_debug("Queue Address (0x%llX, 0x%llX)\n",
206 q_properties
->queue_address
, args
->ring_base_address
);
208 pr_debug("Queue Size (0x%llX, %u)\n",
209 q_properties
->queue_size
, args
->ring_size
);
211 pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
212 (uint64_t) q_properties
->read_ptr
,
213 (uint64_t) q_properties
->write_ptr
);
215 pr_debug("Queue Format (%d)\n", q_properties
->format
);
220 static int kfd_ioctl_create_queue(struct file
*filep
, struct kfd_process
*p
,
223 struct kfd_ioctl_create_queue_args
*args
= data
;
226 unsigned int queue_id
;
227 struct kfd_process_device
*pdd
;
228 struct queue_properties q_properties
;
230 memset(&q_properties
, 0, sizeof(struct queue_properties
));
232 pr_debug("kfd: creating queue ioctl\n");
234 err
= set_queue_properties_from_user(&q_properties
, args
);
238 dev
= kfd_device_by_id(args
->gpu_id
);
242 mutex_lock(&p
->mutex
);
244 pdd
= kfd_bind_process_to_device(dev
, p
);
247 goto err_bind_process
;
250 pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
254 err
= pqm_create_queue(&p
->pqm
, dev
, filep
, &q_properties
, 0,
255 KFD_QUEUE_TYPE_COMPUTE
, &queue_id
);
257 goto err_create_queue
;
259 args
->queue_id
= queue_id
;
261 /* Return gpu_id as doorbell offset for mmap usage */
262 args
->doorbell_offset
= args
->gpu_id
<< PAGE_SHIFT
;
264 mutex_unlock(&p
->mutex
);
266 pr_debug("kfd: queue id %d was created successfully\n", args
->queue_id
);
268 pr_debug("ring buffer address == 0x%016llX\n",
269 args
->ring_base_address
);
271 pr_debug("read ptr address == 0x%016llX\n",
272 args
->read_pointer_address
);
274 pr_debug("write ptr address == 0x%016llX\n",
275 args
->write_pointer_address
);
281 mutex_unlock(&p
->mutex
);
285 static int kfd_ioctl_destroy_queue(struct file
*filp
, struct kfd_process
*p
,
289 struct kfd_ioctl_destroy_queue_args
*args
= data
;
291 pr_debug("kfd: destroying queue id %d for PASID %d\n",
295 mutex_lock(&p
->mutex
);
297 retval
= pqm_destroy_queue(&p
->pqm
, args
->queue_id
);
299 mutex_unlock(&p
->mutex
);
303 static int kfd_ioctl_update_queue(struct file
*filp
, struct kfd_process
*p
,
307 struct kfd_ioctl_update_queue_args
*args
= data
;
308 struct queue_properties properties
;
310 if (args
->queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
311 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
315 if (args
->queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
316 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
320 if ((args
->ring_base_address
) &&
321 (!access_ok(VERIFY_WRITE
,
322 (const void __user
*) args
->ring_base_address
,
323 sizeof(uint64_t)))) {
324 pr_err("kfd: can't access ring base address\n");
328 if (!is_power_of_2(args
->ring_size
) && (args
->ring_size
!= 0)) {
329 pr_err("kfd: ring size must be a power of 2 or 0\n");
333 properties
.queue_address
= args
->ring_base_address
;
334 properties
.queue_size
= args
->ring_size
;
335 properties
.queue_percent
= args
->queue_percentage
;
336 properties
.priority
= args
->queue_priority
;
338 pr_debug("kfd: updating queue id %d for PASID %d\n",
339 args
->queue_id
, p
->pasid
);
341 mutex_lock(&p
->mutex
);
343 retval
= pqm_update_queue(&p
->pqm
, args
->queue_id
, &properties
);
345 mutex_unlock(&p
->mutex
);
350 static int kfd_ioctl_set_memory_policy(struct file
*filep
,
351 struct kfd_process
*p
, void *data
)
353 struct kfd_ioctl_set_memory_policy_args
*args
= data
;
356 struct kfd_process_device
*pdd
;
357 enum cache_policy default_policy
, alternate_policy
;
359 if (args
->default_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
360 && args
->default_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
364 if (args
->alternate_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
365 && args
->alternate_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
369 dev
= kfd_device_by_id(args
->gpu_id
);
373 mutex_lock(&p
->mutex
);
375 pdd
= kfd_bind_process_to_device(dev
, p
);
381 default_policy
= (args
->default_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
382 ? cache_policy_coherent
: cache_policy_noncoherent
;
385 (args
->alternate_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
386 ? cache_policy_coherent
: cache_policy_noncoherent
;
388 if (!dev
->dqm
->set_cache_memory_policy(dev
->dqm
,
392 (void __user
*)args
->alternate_aperture_base
,
393 args
->alternate_aperture_size
))
397 mutex_unlock(&p
->mutex
);
402 static int kfd_ioctl_get_clock_counters(struct file
*filep
,
403 struct kfd_process
*p
, void *data
)
405 struct kfd_ioctl_get_clock_counters_args
*args
= data
;
407 struct timespec time
;
409 dev
= kfd_device_by_id(args
->gpu_id
);
413 /* Reading GPU clock counter from KGD */
414 args
->gpu_clock_counter
= kfd2kgd
->get_gpu_clock_counter(dev
->kgd
);
416 /* No access to rdtsc. Using raw monotonic time */
417 getrawmonotonic(&time
);
418 args
->cpu_clock_counter
= (uint64_t)timespec_to_ns(&time
);
420 get_monotonic_boottime(&time
);
421 args
->system_clock_counter
= (uint64_t)timespec_to_ns(&time
);
423 /* Since the counter is in nano-seconds we use 1GHz frequency */
424 args
->system_clock_freq
= 1000000000;
430 static int kfd_ioctl_get_process_apertures(struct file
*filp
,
431 struct kfd_process
*p
, void *data
)
433 struct kfd_ioctl_get_process_apertures_args
*args
= data
;
434 struct kfd_process_device_apertures
*pAperture
;
435 struct kfd_process_device
*pdd
;
437 dev_dbg(kfd_device
, "get apertures for PASID %d", p
->pasid
);
439 args
->num_of_nodes
= 0;
441 mutex_lock(&p
->mutex
);
443 /*if the process-device list isn't empty*/
444 if (kfd_has_process_device_data(p
)) {
445 /* Run over all pdd of the process */
446 pdd
= kfd_get_first_process_device_data(p
);
449 &args
->process_apertures
[args
->num_of_nodes
];
450 pAperture
->gpu_id
= pdd
->dev
->id
;
451 pAperture
->lds_base
= pdd
->lds_base
;
452 pAperture
->lds_limit
= pdd
->lds_limit
;
453 pAperture
->gpuvm_base
= pdd
->gpuvm_base
;
454 pAperture
->gpuvm_limit
= pdd
->gpuvm_limit
;
455 pAperture
->scratch_base
= pdd
->scratch_base
;
456 pAperture
->scratch_limit
= pdd
->scratch_limit
;
459 "node id %u\n", args
->num_of_nodes
);
461 "gpu id %u\n", pdd
->dev
->id
);
463 "lds_base %llX\n", pdd
->lds_base
);
465 "lds_limit %llX\n", pdd
->lds_limit
);
467 "gpuvm_base %llX\n", pdd
->gpuvm_base
);
469 "gpuvm_limit %llX\n", pdd
->gpuvm_limit
);
471 "scratch_base %llX\n", pdd
->scratch_base
);
473 "scratch_limit %llX\n", pdd
->scratch_limit
);
475 args
->num_of_nodes
++;
476 } while ((pdd
= kfd_get_next_process_device_data(p
, pdd
)) != NULL
&&
477 (args
->num_of_nodes
< NUM_OF_SUPPORTED_GPUS
));
480 mutex_unlock(&p
->mutex
);
485 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
486 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl}
489 static const struct amdkfd_ioctl_desc amdkfd_ioctls
[] = {
490 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION
,
491 kfd_ioctl_get_version
, 0),
493 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE
,
494 kfd_ioctl_create_queue
, 0),
496 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE
,
497 kfd_ioctl_destroy_queue
, 0),
499 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY
,
500 kfd_ioctl_set_memory_policy
, 0),
502 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS
,
503 kfd_ioctl_get_clock_counters
, 0),
505 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES
,
506 kfd_ioctl_get_process_apertures
, 0),
508 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE
,
509 kfd_ioctl_update_queue
, 0),
512 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
514 static long kfd_ioctl(struct file
*filep
, unsigned int cmd
, unsigned long arg
)
516 struct kfd_process
*process
;
517 amdkfd_ioctl_t
*func
;
518 const struct amdkfd_ioctl_desc
*ioctl
= NULL
;
519 unsigned int nr
= _IOC_NR(cmd
);
520 char stack_kdata
[128];
522 unsigned int usize
, asize
;
523 int retcode
= -EINVAL
;
525 if (nr
>= AMDKFD_CORE_IOCTL_COUNT
)
528 if ((nr
>= AMDKFD_COMMAND_START
) && (nr
< AMDKFD_COMMAND_END
)) {
531 ioctl
= &amdkfd_ioctls
[nr
];
533 amdkfd_size
= _IOC_SIZE(ioctl
->cmd
);
534 usize
= asize
= _IOC_SIZE(cmd
);
535 if (amdkfd_size
> asize
)
542 dev_dbg(kfd_device
, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd
, nr
, arg
);
544 process
= kfd_get_process(current
);
545 if (IS_ERR(process
)) {
546 dev_dbg(kfd_device
, "no process\n");
550 /* Do not trust userspace, use our own definition */
553 if (unlikely(!func
)) {
554 dev_dbg(kfd_device
, "no function\n");
559 if (cmd
& (IOC_IN
| IOC_OUT
)) {
560 if (asize
<= sizeof(stack_kdata
)) {
563 kdata
= kmalloc(asize
, GFP_KERNEL
);
570 memset(kdata
+ usize
, 0, asize
- usize
);
574 if (copy_from_user(kdata
, (void __user
*)arg
, usize
) != 0) {
578 } else if (cmd
& IOC_OUT
) {
579 memset(kdata
, 0, usize
);
582 retcode
= func(filep
, process
, kdata
);
585 if (copy_to_user((void __user
*)arg
, kdata
, usize
) != 0)
590 dev_dbg(kfd_device
, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
591 task_pid_nr(current
), cmd
, nr
);
593 if (kdata
!= stack_kdata
)
597 dev_dbg(kfd_device
, "ret = %d\n", retcode
);
602 static int kfd_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
604 struct kfd_process
*process
;
606 process
= kfd_get_process(current
);
608 return PTR_ERR(process
);
610 return kfd_doorbell_mmap(process
, vma
);