Merge remote-tracking branches 'spi/topic/img-spfi', 'spi/topic/imx', 'spi/topic...
[deliverable/linux.git] / drivers / gpu / drm / amd / amdkfd / kfd_process.c
CommitLineData
19f6d2a6
OG
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/mutex.h>
24#include <linux/log2.h>
25#include <linux/sched.h>
26#include <linux/slab.h>
b17f068a 27#include <linux/amd-iommu.h>
19f6d2a6 28#include <linux/notifier.h>
dd59239a
AS
29#include <linux/compat.h>
30
19f6d2a6
OG
31struct mm_struct;
32
33#include "kfd_priv.h"
34
35/*
36 * Initial size for the array of queues.
37 * The allocated size is doubled each time
38 * it is exceeded up to MAX_PROCESS_QUEUES.
39 */
40#define INITIAL_QUEUE_ARRAY_SIZE 16
41
42/*
43 * List of struct kfd_process (field kfd_process).
44 * Unique/indexed by mm_struct*
45 */
46#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
47static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
48static DEFINE_MUTEX(kfd_processes_mutex);
49
50DEFINE_STATIC_SRCU(kfd_processes_srcu);
51
52static struct workqueue_struct *kfd_process_wq;
53
54struct kfd_process_release_work {
55 struct work_struct kfd_work;
56 struct kfd_process *p;
57};
58
59static struct kfd_process *find_process(const struct task_struct *thread);
60static struct kfd_process *create_process(const struct task_struct *thread);
61
62void kfd_process_create_wq(void)
63{
64 if (!kfd_process_wq)
65 kfd_process_wq = create_workqueue("kfd_process_wq");
66}
67
68void kfd_process_destroy_wq(void)
69{
70 if (kfd_process_wq) {
71 flush_workqueue(kfd_process_wq);
72 destroy_workqueue(kfd_process_wq);
73 kfd_process_wq = NULL;
74 }
75}
76
77struct kfd_process *kfd_create_process(const struct task_struct *thread)
78{
79 struct kfd_process *process;
80
81 BUG_ON(!kfd_process_wq);
82
83 if (thread->mm == NULL)
84 return ERR_PTR(-EINVAL);
85
86 /* Only the pthreads threading model is supported. */
87 if (thread->group_leader->mm != thread->mm)
88 return ERR_PTR(-EINVAL);
89
90 /* Take mmap_sem because we call __mmu_notifier_register inside */
91 down_write(&thread->mm->mmap_sem);
92
93 /*
94 * take kfd processes mutex before starting of process creation
95 * so there won't be a case where two threads of the same process
96 * create two kfd_process structures
97 */
98 mutex_lock(&kfd_processes_mutex);
99
100 /* A prior open of /dev/kfd could have already created the process. */
101 process = find_process(thread);
102 if (process)
103 pr_debug("kfd: process already found\n");
104
105 if (!process)
106 process = create_process(thread);
107
108 mutex_unlock(&kfd_processes_mutex);
109
110 up_write(&thread->mm->mmap_sem);
111
112 return process;
113}
114
115struct kfd_process *kfd_get_process(const struct task_struct *thread)
116{
117 struct kfd_process *process;
118
119 if (thread->mm == NULL)
120 return ERR_PTR(-EINVAL);
121
122 /* Only the pthreads threading model is supported. */
123 if (thread->group_leader->mm != thread->mm)
124 return ERR_PTR(-EINVAL);
125
126 process = find_process(thread);
127
128 return process;
129}
130
131static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
132{
133 struct kfd_process *process;
134
135 hash_for_each_possible_rcu(kfd_processes_table, process,
136 kfd_processes, (uintptr_t)mm)
137 if (process->mm == mm)
138 return process;
139
140 return NULL;
141}
142
143static struct kfd_process *find_process(const struct task_struct *thread)
144{
145 struct kfd_process *p;
146 int idx;
147
148 idx = srcu_read_lock(&kfd_processes_srcu);
149 p = find_process_by_mm(thread->mm);
150 srcu_read_unlock(&kfd_processes_srcu, idx);
151
152 return p;
153}
154
155static void kfd_process_wq_release(struct work_struct *work)
156{
157 struct kfd_process_release_work *my_work;
158 struct kfd_process_device *pdd, *temp;
159 struct kfd_process *p;
160
161 my_work = (struct kfd_process_release_work *) work;
162
163 p = my_work->p;
164
165 mutex_lock(&p->mutex);
166
167 list_for_each_entry_safe(pdd, temp, &p->per_device_data,
168 per_device_list) {
b17f068a 169 amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
19f6d2a6
OG
170 list_del(&pdd->per_device_list);
171
172 kfree(pdd);
173 }
174
175 kfd_pasid_free(p->pasid);
176
177 mutex_unlock(&p->mutex);
178
179 mutex_destroy(&p->mutex);
180
181 kfree(p->queues);
182
183 kfree(p);
184
185 kfree((void *)work);
186}
187
188static void kfd_process_destroy_delayed(struct rcu_head *rcu)
189{
190 struct kfd_process_release_work *work;
191 struct kfd_process *p;
192
193 BUG_ON(!kfd_process_wq);
194
195 p = container_of(rcu, struct kfd_process, rcu);
196 BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
197
198 mmdrop(p->mm);
199
200 work = (struct kfd_process_release_work *)
c448a142 201 kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
19f6d2a6
OG
202
203 if (work) {
204 INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
205 work->p = p;
206 queue_work(kfd_process_wq, (struct work_struct *) work);
207 }
208}
209
210static void kfd_process_notifier_release(struct mmu_notifier *mn,
211 struct mm_struct *mm)
212{
213 struct kfd_process *p;
214
215 /*
216 * The kfd_process structure can not be free because the
217 * mmu_notifier srcu is read locked
218 */
219 p = container_of(mn, struct kfd_process, mmu_notifier);
220 BUG_ON(p->mm != mm);
221
222 mutex_lock(&kfd_processes_mutex);
223 hash_del_rcu(&p->kfd_processes);
224 mutex_unlock(&kfd_processes_mutex);
225 synchronize_srcu(&kfd_processes_srcu);
226
45102048
BG
227 mutex_lock(&p->mutex);
228
229 /* In case our notifier is called before IOMMU notifier */
230 pqm_uninit(&p->pqm);
231
232 mutex_unlock(&p->mutex);
233
19f6d2a6
OG
234 /*
235 * Because we drop mm_count inside kfd_process_destroy_delayed
236 * and because the mmu_notifier_unregister function also drop
237 * mm_count we need to take an extra count here.
238 */
239 atomic_inc(&p->mm->mm_count);
240 mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
241 mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
242}
243
244static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
245 .release = kfd_process_notifier_release,
246};
247
248static struct kfd_process *create_process(const struct task_struct *thread)
249{
250 struct kfd_process *process;
251 int err = -ENOMEM;
252
253 process = kzalloc(sizeof(*process), GFP_KERNEL);
254
255 if (!process)
256 goto err_alloc_process;
257
258 process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
259 sizeof(process->queues[0]), GFP_KERNEL);
260 if (!process->queues)
261 goto err_alloc_queues;
262
263 process->pasid = kfd_pasid_alloc();
264 if (process->pasid == 0)
265 goto err_alloc_pasid;
266
267 mutex_init(&process->mutex);
268
269 process->mm = thread->mm;
270
271 /* register notifier */
272 process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
273 err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
274 if (err)
275 goto err_mmu_notifier;
276
277 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
278 (uintptr_t)process->mm);
279
280 process->lead_thread = thread->group_leader;
281
282 process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
283
284 INIT_LIST_HEAD(&process->per_device_data);
285
45102048
BG
286 err = pqm_init(&process->pqm, process);
287 if (err != 0)
288 goto err_process_pqm_init;
289
dd59239a
AS
290 /* init process apertures*/
291 process->is_32bit_user_mode = is_compat_task();
292 if (kfd_init_apertures(process) != 0)
293 goto err_init_apretures;
294
19f6d2a6
OG
295 return process;
296
dd59239a
AS
297err_init_apretures:
298 pqm_uninit(&process->pqm);
45102048
BG
299err_process_pqm_init:
300 hash_del_rcu(&process->kfd_processes);
301 synchronize_rcu();
302 mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
19f6d2a6
OG
303err_mmu_notifier:
304 kfd_pasid_free(process->pasid);
305err_alloc_pasid:
306 kfree(process->queues);
307err_alloc_queues:
308 kfree(process);
309err_alloc_process:
310 return ERR_PTR(err);
311}
312
313struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
314 struct kfd_process *p,
315 int create_pdd)
316{
317 struct kfd_process_device *pdd = NULL;
318
319 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
320 if (pdd->dev == dev)
321 return pdd;
322
323 if (create_pdd) {
324 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
325 if (pdd != NULL) {
326 pdd->dev = dev;
45102048
BG
327 INIT_LIST_HEAD(&pdd->qpd.queues_list);
328 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
329 pdd->qpd.dqm = dev->dqm;
19f6d2a6
OG
330 list_add(&pdd->per_device_list, &p->per_device_data);
331 }
332 }
333
334 return pdd;
335}
336
337/*
338 * Direct the IOMMU to bind the process (specifically the pasid->mm)
339 * to the device.
340 * Unbinding occurs when the process dies or the device is removed.
341 *
342 * Assumes that the process lock is held.
343 */
344struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
345 struct kfd_process *p)
346{
347 struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1);
b17f068a 348 int err;
19f6d2a6
OG
349
350 if (pdd == NULL)
351 return ERR_PTR(-ENOMEM);
352
353 if (pdd->bound)
354 return pdd;
355
b17f068a
OG
356 err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
357 if (err < 0)
358 return ERR_PTR(err);
359
19f6d2a6
OG
360 pdd->bound = true;
361
362 return pdd;
363}
364
365void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
366{
367 struct kfd_process *p;
368 struct kfd_process_device *pdd;
369 int idx, i;
370
371 BUG_ON(dev == NULL);
372
373 idx = srcu_read_lock(&kfd_processes_srcu);
374
375 hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
376 if (p->pasid == pasid)
377 break;
378
379 srcu_read_unlock(&kfd_processes_srcu, idx);
380
381 BUG_ON(p->pasid != pasid);
382
383 mutex_lock(&p->mutex);
384
45102048
BG
385 pqm_uninit(&p->pqm);
386
19f6d2a6
OG
387 pdd = kfd_get_process_device_data(dev, p, 0);
388
389 /*
390 * Just mark pdd as unbound, because we still need it to call
391 * amd_iommu_unbind_pasid() in when the process exits.
392 * We don't call amd_iommu_unbind_pasid() here
393 * because the IOMMU called us.
394 */
395 if (pdd)
396 pdd->bound = false;
397
398 mutex_unlock(&p->mutex);
399}
400
401struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
402{
403 return list_first_entry(&p->per_device_data,
404 struct kfd_process_device,
405 per_device_list);
406}
407
408struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
409 struct kfd_process_device *pdd)
410{
411 if (list_is_last(&pdd->per_device_list, &p->per_device_data))
412 return NULL;
413 return list_next_entry(pdd, per_device_list);
414}
415
416bool kfd_has_process_device_data(struct kfd_process *p)
417{
418 return !(list_empty(&p->per_device_data));
419}
This page took 0.051864 seconds and 5 git commands to generate.