2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/device.h>
24 #include <linux/export.h>
25 #include <linux/err.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
29 #include <linux/uaccess.h>
30 #include <linux/compat.h>
31 #include <uapi/linux/kfd_ioctl.h>
32 #include <linux/time.h>
34 #include <linux/uaccess.h>
35 #include <uapi/asm-generic/mman-common.h>
36 #include <asm/processor.h>
38 #include "kfd_device_queue_manager.h"
40 static long kfd_ioctl(struct file
*, unsigned int, unsigned long);
41 static int kfd_open(struct inode
*, struct file
*);
42 static int kfd_mmap(struct file
*, struct vm_area_struct
*);
44 static const char kfd_dev_name
[] = "kfd";
46 static const struct file_operations kfd_fops
= {
48 .unlocked_ioctl
= kfd_ioctl
,
49 .compat_ioctl
= kfd_ioctl
,
54 static int kfd_char_dev_major
= -1;
55 static struct class *kfd_class
;
56 struct device
*kfd_device
;
58 int kfd_chardev_init(void)
62 kfd_char_dev_major
= register_chrdev(0, kfd_dev_name
, &kfd_fops
);
63 err
= kfd_char_dev_major
;
65 goto err_register_chrdev
;
67 kfd_class
= class_create(THIS_MODULE
, kfd_dev_name
);
68 err
= PTR_ERR(kfd_class
);
69 if (IS_ERR(kfd_class
))
70 goto err_class_create
;
72 kfd_device
= device_create(kfd_class
, NULL
,
73 MKDEV(kfd_char_dev_major
, 0),
75 err
= PTR_ERR(kfd_device
);
76 if (IS_ERR(kfd_device
))
77 goto err_device_create
;
82 class_destroy(kfd_class
);
84 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
89 void kfd_chardev_exit(void)
91 device_destroy(kfd_class
, MKDEV(kfd_char_dev_major
, 0));
92 class_destroy(kfd_class
);
93 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
96 struct device
*kfd_chardev(void)
102 static int kfd_open(struct inode
*inode
, struct file
*filep
)
104 struct kfd_process
*process
;
105 bool is_32bit_user_mode
;
107 if (iminor(inode
) != 0)
110 is_32bit_user_mode
= is_compat_task();
112 if (is_32bit_user_mode
== true) {
114 "Process %d (32-bit) failed to open /dev/kfd\n"
115 "32-bit processes are not supported by amdkfd\n",
120 process
= kfd_create_process(current
);
122 return PTR_ERR(process
);
124 process
->is_32bit_user_mode
= is_32bit_user_mode
;
126 dev_dbg(kfd_device
, "process %d opened, compat mode (32 bit) - %d\n",
127 process
->pasid
, process
->is_32bit_user_mode
);
129 kfd_init_apertures(process
);
134 static long kfd_ioctl_get_version(struct file
*filep
, struct kfd_process
*p
,
137 struct kfd_ioctl_get_version_args args
;
140 args
.major_version
= KFD_IOCTL_MAJOR_VERSION
;
141 args
.minor_version
= KFD_IOCTL_MINOR_VERSION
;
143 if (copy_to_user(arg
, &args
, sizeof(args
)))
149 static int set_queue_properties_from_user(struct queue_properties
*q_properties
,
150 struct kfd_ioctl_create_queue_args
*args
)
152 if (args
->queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
153 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
157 if (args
->queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
158 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
162 if ((args
->ring_base_address
) &&
163 (!access_ok(VERIFY_WRITE
,
164 (const void __user
*) args
->ring_base_address
,
165 sizeof(uint64_t)))) {
166 pr_err("kfd: can't access ring base address\n");
170 if (!is_power_of_2(args
->ring_size
) && (args
->ring_size
!= 0)) {
171 pr_err("kfd: ring size must be a power of 2 or 0\n");
175 if (!access_ok(VERIFY_WRITE
,
176 (const void __user
*) args
->read_pointer_address
,
178 pr_err("kfd: can't access read pointer\n");
182 if (!access_ok(VERIFY_WRITE
,
183 (const void __user
*) args
->write_pointer_address
,
185 pr_err("kfd: can't access write pointer\n");
189 q_properties
->is_interop
= false;
190 q_properties
->queue_percent
= args
->queue_percentage
;
191 q_properties
->priority
= args
->queue_priority
;
192 q_properties
->queue_address
= args
->ring_base_address
;
193 q_properties
->queue_size
= args
->ring_size
;
194 q_properties
->read_ptr
= (uint32_t *) args
->read_pointer_address
;
195 q_properties
->write_ptr
= (uint32_t *) args
->write_pointer_address
;
196 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE
||
197 args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
198 q_properties
->type
= KFD_QUEUE_TYPE_COMPUTE
;
202 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
203 q_properties
->format
= KFD_QUEUE_FORMAT_AQL
;
205 q_properties
->format
= KFD_QUEUE_FORMAT_PM4
;
207 pr_debug("Queue Percentage (%d, %d)\n",
208 q_properties
->queue_percent
, args
->queue_percentage
);
210 pr_debug("Queue Priority (%d, %d)\n",
211 q_properties
->priority
, args
->queue_priority
);
213 pr_debug("Queue Address (0x%llX, 0x%llX)\n",
214 q_properties
->queue_address
, args
->ring_base_address
);
216 pr_debug("Queue Size (0x%llX, %u)\n",
217 q_properties
->queue_size
, args
->ring_size
);
219 pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
220 (uint64_t) q_properties
->read_ptr
,
221 (uint64_t) q_properties
->write_ptr
);
223 pr_debug("Queue Format (%d)\n", q_properties
->format
);
228 static long kfd_ioctl_create_queue(struct file
*filep
, struct kfd_process
*p
,
231 struct kfd_ioctl_create_queue_args args
;
234 unsigned int queue_id
;
235 struct kfd_process_device
*pdd
;
236 struct queue_properties q_properties
;
238 memset(&q_properties
, 0, sizeof(struct queue_properties
));
240 if (copy_from_user(&args
, arg
, sizeof(args
)))
243 pr_debug("kfd: creating queue ioctl\n");
245 err
= set_queue_properties_from_user(&q_properties
, &args
);
249 dev
= kfd_device_by_id(args
.gpu_id
);
253 mutex_lock(&p
->mutex
);
255 pdd
= kfd_bind_process_to_device(dev
, p
);
258 goto err_bind_process
;
261 pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
265 err
= pqm_create_queue(&p
->pqm
, dev
, filep
, &q_properties
, 0,
266 KFD_QUEUE_TYPE_COMPUTE
, &queue_id
);
268 goto err_create_queue
;
270 args
.queue_id
= queue_id
;
272 /* Return gpu_id as doorbell offset for mmap usage */
273 args
.doorbell_offset
= args
.gpu_id
<< PAGE_SHIFT
;
275 if (copy_to_user(arg
, &args
, sizeof(args
))) {
277 goto err_copy_args_out
;
280 mutex_unlock(&p
->mutex
);
282 pr_debug("kfd: queue id %d was created successfully\n", args
.queue_id
);
284 pr_debug("ring buffer address == 0x%016llX\n",
285 args
.ring_base_address
);
287 pr_debug("read ptr address == 0x%016llX\n",
288 args
.read_pointer_address
);
290 pr_debug("write ptr address == 0x%016llX\n",
291 args
.write_pointer_address
);
296 pqm_destroy_queue(&p
->pqm
, queue_id
);
299 mutex_unlock(&p
->mutex
);
303 static int kfd_ioctl_destroy_queue(struct file
*filp
, struct kfd_process
*p
,
307 struct kfd_ioctl_destroy_queue_args args
;
309 if (copy_from_user(&args
, arg
, sizeof(args
)))
312 pr_debug("kfd: destroying queue id %d for PASID %d\n",
316 mutex_lock(&p
->mutex
);
318 retval
= pqm_destroy_queue(&p
->pqm
, args
.queue_id
);
320 mutex_unlock(&p
->mutex
);
324 static int kfd_ioctl_update_queue(struct file
*filp
, struct kfd_process
*p
,
328 struct kfd_ioctl_update_queue_args args
;
329 struct queue_properties properties
;
331 if (copy_from_user(&args
, arg
, sizeof(args
)))
334 if (args
.queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
335 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
339 if (args
.queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
340 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
344 if ((args
.ring_base_address
) &&
345 (!access_ok(VERIFY_WRITE
,
346 (const void __user
*) args
.ring_base_address
,
347 sizeof(uint64_t)))) {
348 pr_err("kfd: can't access ring base address\n");
352 if (!is_power_of_2(args
.ring_size
) && (args
.ring_size
!= 0)) {
353 pr_err("kfd: ring size must be a power of 2 or 0\n");
357 properties
.queue_address
= args
.ring_base_address
;
358 properties
.queue_size
= args
.ring_size
;
359 properties
.queue_percent
= args
.queue_percentage
;
360 properties
.priority
= args
.queue_priority
;
362 pr_debug("kfd: updating queue id %d for PASID %d\n",
363 args
.queue_id
, p
->pasid
);
365 mutex_lock(&p
->mutex
);
367 retval
= pqm_update_queue(&p
->pqm
, args
.queue_id
, &properties
);
369 mutex_unlock(&p
->mutex
);
374 static long kfd_ioctl_set_memory_policy(struct file
*filep
,
375 struct kfd_process
*p
, void __user
*arg
)
377 struct kfd_ioctl_set_memory_policy_args args
;
380 struct kfd_process_device
*pdd
;
381 enum cache_policy default_policy
, alternate_policy
;
383 if (copy_from_user(&args
, arg
, sizeof(args
)))
386 if (args
.default_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
387 && args
.default_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
391 if (args
.alternate_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
392 && args
.alternate_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
396 dev
= kfd_device_by_id(args
.gpu_id
);
400 mutex_lock(&p
->mutex
);
402 pdd
= kfd_bind_process_to_device(dev
, p
);
408 default_policy
= (args
.default_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
409 ? cache_policy_coherent
: cache_policy_noncoherent
;
412 (args
.alternate_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
413 ? cache_policy_coherent
: cache_policy_noncoherent
;
415 if (!dev
->dqm
->set_cache_memory_policy(dev
->dqm
,
419 (void __user
*)args
.alternate_aperture_base
,
420 args
.alternate_aperture_size
))
424 mutex_unlock(&p
->mutex
);
429 static long kfd_ioctl_get_clock_counters(struct file
*filep
,
430 struct kfd_process
*p
, void __user
*arg
)
432 struct kfd_ioctl_get_clock_counters_args args
;
434 struct timespec time
;
436 if (copy_from_user(&args
, arg
, sizeof(args
)))
439 dev
= kfd_device_by_id(args
.gpu_id
);
443 /* Reading GPU clock counter from KGD */
444 args
.gpu_clock_counter
= kfd2kgd
->get_gpu_clock_counter(dev
->kgd
);
446 /* No access to rdtsc. Using raw monotonic time */
447 getrawmonotonic(&time
);
448 args
.cpu_clock_counter
= (uint64_t)timespec_to_ns(&time
);
450 get_monotonic_boottime(&time
);
451 args
.system_clock_counter
= (uint64_t)timespec_to_ns(&time
);
453 /* Since the counter is in nano-seconds we use 1GHz frequency */
454 args
.system_clock_freq
= 1000000000;
456 if (copy_to_user(arg
, &args
, sizeof(args
)))
463 static int kfd_ioctl_get_process_apertures(struct file
*filp
,
464 struct kfd_process
*p
, void __user
*arg
)
466 struct kfd_ioctl_get_process_apertures_args args
;
467 struct kfd_process_device_apertures
*pAperture
;
468 struct kfd_process_device
*pdd
;
470 dev_dbg(kfd_device
, "get apertures for PASID %d", p
->pasid
);
472 if (copy_from_user(&args
, arg
, sizeof(args
)))
475 args
.num_of_nodes
= 0;
477 mutex_lock(&p
->mutex
);
479 /*if the process-device list isn't empty*/
480 if (kfd_has_process_device_data(p
)) {
481 /* Run over all pdd of the process */
482 pdd
= kfd_get_first_process_device_data(p
);
484 pAperture
= &args
.process_apertures
[args
.num_of_nodes
];
485 pAperture
->gpu_id
= pdd
->dev
->id
;
486 pAperture
->lds_base
= pdd
->lds_base
;
487 pAperture
->lds_limit
= pdd
->lds_limit
;
488 pAperture
->gpuvm_base
= pdd
->gpuvm_base
;
489 pAperture
->gpuvm_limit
= pdd
->gpuvm_limit
;
490 pAperture
->scratch_base
= pdd
->scratch_base
;
491 pAperture
->scratch_limit
= pdd
->scratch_limit
;
494 "node id %u\n", args
.num_of_nodes
);
496 "gpu id %u\n", pdd
->dev
->id
);
498 "lds_base %llX\n", pdd
->lds_base
);
500 "lds_limit %llX\n", pdd
->lds_limit
);
502 "gpuvm_base %llX\n", pdd
->gpuvm_base
);
504 "gpuvm_limit %llX\n", pdd
->gpuvm_limit
);
506 "scratch_base %llX\n", pdd
->scratch_base
);
508 "scratch_limit %llX\n", pdd
->scratch_limit
);
511 } while ((pdd
= kfd_get_next_process_device_data(p
, pdd
)) != NULL
&&
512 (args
.num_of_nodes
< NUM_OF_SUPPORTED_GPUS
));
515 mutex_unlock(&p
->mutex
);
517 if (copy_to_user(arg
, &args
, sizeof(args
)))
523 static long kfd_ioctl(struct file
*filep
, unsigned int cmd
, unsigned long arg
)
525 struct kfd_process
*process
;
529 "ioctl cmd 0x%x (#%d), arg 0x%lx\n",
530 cmd
, _IOC_NR(cmd
), arg
);
532 process
= kfd_get_process(current
);
534 return PTR_ERR(process
);
537 case KFD_IOC_GET_VERSION
:
538 err
= kfd_ioctl_get_version(filep
, process
, (void __user
*)arg
);
540 case KFD_IOC_CREATE_QUEUE
:
541 err
= kfd_ioctl_create_queue(filep
, process
,
545 case KFD_IOC_DESTROY_QUEUE
:
546 err
= kfd_ioctl_destroy_queue(filep
, process
,
550 case KFD_IOC_SET_MEMORY_POLICY
:
551 err
= kfd_ioctl_set_memory_policy(filep
, process
,
555 case KFD_IOC_GET_CLOCK_COUNTERS
:
556 err
= kfd_ioctl_get_clock_counters(filep
, process
,
560 case KFD_IOC_GET_PROCESS_APERTURES
:
561 err
= kfd_ioctl_get_process_apertures(filep
, process
,
565 case KFD_IOC_UPDATE_QUEUE
:
566 err
= kfd_ioctl_update_queue(filep
, process
,
572 "unknown ioctl cmd 0x%x, arg 0x%lx)\n",
580 "ioctl error %ld for ioctl cmd 0x%x (#%d)\n",
581 err
, cmd
, _IOC_NR(cmd
));
586 static int kfd_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
588 struct kfd_process
*process
;
590 process
= kfd_get_process(current
);
592 return PTR_ERR(process
);
594 return kfd_doorbell_mmap(process
, vma
);