[deliverable/linux.git] / drivers / gpu / drm / amd / amdkfd / kfd_process.c

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#include <linux/mutex.h>
#include <linux/log2.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/amd-iommu.h>
#include <linux/notifier.h>
#include <linux/compat.h>

struct mm_struct;

#include "kfd_priv.h"
#include "kfd_dbgmgr.h"

/*
 * Initial size for the array of queues.
 * The allocated size is doubled each time
 * it is exceeded up to MAX_PROCESS_QUEUES.
 */
#define INITIAL_QUEUE_ARRAY_SIZE 16

/*
 * List of struct kfd_process (field kfd_process).
 * Unique/indexed by mm_struct*
 */
#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
static DEFINE_MUTEX(kfd_processes_mutex);

DEFINE_STATIC_SRCU(kfd_processes_srcu);

static struct workqueue_struct *kfd_process_wq;

struct kfd_process_release_work {
	struct work_struct kfd_work;
	struct kfd_process *p;
};

static struct kfd_process *find_process(const struct task_struct *thread);
static struct kfd_process *create_process(const struct task_struct *thread);

void kfd_process_create_wq(void)
{
	if (!kfd_process_wq)
		kfd_process_wq = create_workqueue("kfd_process_wq");
}

void kfd_process_destroy_wq(void)
{
	if (kfd_process_wq) {
		flush_workqueue(kfd_process_wq);
		destroy_workqueue(kfd_process_wq);
		kfd_process_wq = NULL;
	}
}

struct kfd_process *kfd_create_process(const struct task_struct *thread)
{
	struct kfd_process *process;

	BUG_ON(!kfd_process_wq);

	if (thread->mm == NULL)
		return ERR_PTR(-EINVAL);

	/* Only the pthreads threading model is supported. */
	if (thread->group_leader->mm != thread->mm)
		return ERR_PTR(-EINVAL);

	/* Take mmap_sem because we call __mmu_notifier_register inside */
	down_write(&thread->mm->mmap_sem);

	/*
	 * take kfd processes mutex before starting of process creation
	 * so there won't be a case where two threads of the same process
	 * create two kfd_process structures
	 */
	mutex_lock(&kfd_processes_mutex);

	/* A prior open of /dev/kfd could have already created the process. */
	process = find_process(thread);
	if (process)
		pr_debug("kfd: process already found\n");

	if (!process)
		process = create_process(thread);

	mutex_unlock(&kfd_processes_mutex);

	up_write(&thread->mm->mmap_sem);

	return process;
}

struct kfd_process *kfd_get_process(const struct task_struct *thread)
{
	struct kfd_process *process;

	if (thread->mm == NULL)
		return ERR_PTR(-EINVAL);

	/* Only the pthreads threading model is supported. */
	if (thread->group_leader->mm != thread->mm)
		return ERR_PTR(-EINVAL);

	process = find_process(thread);

	return process;
}

static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
{
	struct kfd_process *process;

	hash_for_each_possible_rcu(kfd_processes_table, process,
					kfd_processes, (uintptr_t)mm)
		if (process->mm == mm)
			return process;

	return NULL;
}

static struct kfd_process *find_process(const struct task_struct *thread)
{
	struct kfd_process *p;
	int idx;

	idx = srcu_read_lock(&kfd_processes_srcu);
	p = find_process_by_mm(thread->mm);
	srcu_read_unlock(&kfd_processes_srcu, idx);

	return p;
}

static void kfd_process_wq_release(struct work_struct *work)
{
	struct kfd_process_release_work *my_work;
	struct kfd_process_device *pdd, *temp;
	struct kfd_process *p;

	my_work = (struct kfd_process_release_work *) work;

	p = my_work->p;

	pr_debug("Releasing process (pasid %d) in workqueue\n",
			p->pasid);

	mutex_lock(&p->mutex);

	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
							per_device_list) {
		pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
				pdd->dev->id, p->pasid);

		if (pdd->reset_wavefronts)
			dbgdev_wave_reset_wavefronts(pdd->dev, p);

		amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
		list_del(&pdd->per_device_list);

		kfree(pdd);
	}

	kfd_event_free_process(p);

	kfd_pasid_free(p->pasid);

	mutex_unlock(&p->mutex);

	mutex_destroy(&p->mutex);

	kfree(p->queues);

	kfree(p);

	kfree(work);
}

static void kfd_process_destroy_delayed(struct rcu_head *rcu)
{
	struct kfd_process_release_work *work;
	struct kfd_process *p;

	BUG_ON(!kfd_process_wq);

	p = container_of(rcu, struct kfd_process, rcu);
	BUG_ON(atomic_read(&p->mm->mm_count) <= 0);

	mmdrop(p->mm);

	work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);

	if (work) {
		INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
		work->p = p;
		queue_work(kfd_process_wq, (struct work_struct *) work);
	}
}

static void kfd_process_notifier_release(struct mmu_notifier *mn,
					struct mm_struct *mm)
{
	struct kfd_process *p;
	struct kfd_process_device *pdd = NULL;

	/*
	 * The kfd_process structure can not be free because the
	 * mmu_notifier srcu is read locked
	 */
	p = container_of(mn, struct kfd_process, mmu_notifier);
	BUG_ON(p->mm != mm);

	mutex_lock(&kfd_processes_mutex);
	hash_del_rcu(&p->kfd_processes);
	mutex_unlock(&kfd_processes_mutex);
	synchronize_srcu(&kfd_processes_srcu);

	mutex_lock(&p->mutex);

	/* In case our notifier is called before IOMMU notifier */
	pqm_uninit(&p->pqm);

	/* Iterate over all process device data structure and check
	 * if we should delete debug managers and reset all wavefronts
	 */
	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
		if ((pdd->dev->dbgmgr) &&
				(pdd->dev->dbgmgr->pasid == p->pasid))
			kfd_dbgmgr_destroy(pdd->dev->dbgmgr);

		if (pdd->reset_wavefronts) {
			pr_warn("amdkfd: Resetting all wave fronts\n");
			dbgdev_wave_reset_wavefronts(pdd->dev, p);
			pdd->reset_wavefronts = false;
		}
	}

	mutex_unlock(&p->mutex);

	/*
	 * Because we drop mm_count inside kfd_process_destroy_delayed
	 * and because the mmu_notifier_unregister function also drop
	 * mm_count we need to take an extra count here.
	 */
	atomic_inc(&p->mm->mm_count);
	mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
	mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
}

static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
	.release = kfd_process_notifier_release,
};

static struct kfd_process *create_process(const struct task_struct *thread)
{
	struct kfd_process *process;
	int err = -ENOMEM;

	process = kzalloc(sizeof(*process), GFP_KERNEL);

	if (!process)
		goto err_alloc_process;

	process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
					sizeof(process->queues[0]), GFP_KERNEL);
	if (!process->queues)
		goto err_alloc_queues;

	process->pasid = kfd_pasid_alloc();
	if (process->pasid == 0)
		goto err_alloc_pasid;

	mutex_init(&process->mutex);

	process->mm = thread->mm;

	/* register notifier */
	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
	err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
	if (err)
		goto err_mmu_notifier;

	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
			(uintptr_t)process->mm);

	process->lead_thread = thread->group_leader;

	process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;

	INIT_LIST_HEAD(&process->per_device_data);

	kfd_event_init_process(process);

	err = pqm_init(&process->pqm, process);
	if (err != 0)
		goto err_process_pqm_init;

	/* init process apertures*/
	process->is_32bit_user_mode = in_compat_syscall();
	if (kfd_init_apertures(process) != 0)
		goto err_init_apretures;

	return process;

err_init_apretures:
	pqm_uninit(&process->pqm);
err_process_pqm_init:
	hash_del_rcu(&process->kfd_processes);
	synchronize_rcu();
	mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
err_mmu_notifier:
	kfd_pasid_free(process->pasid);
err_alloc_pasid:
	kfree(process->queues);
err_alloc_queues:
	kfree(process);
err_alloc_process:
	return ERR_PTR(err);
}

struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
							struct kfd_process *p)
{
	struct kfd_process_device *pdd = NULL;

	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
		if (pdd->dev == dev)
			break;

	return pdd;
}

struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
							struct kfd_process *p)
{
	struct kfd_process_device *pdd = NULL;

	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
	if (pdd != NULL) {
		pdd->dev = dev;
		INIT_LIST_HEAD(&pdd->qpd.queues_list);
		INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
		pdd->qpd.dqm = dev->dqm;
		pdd->reset_wavefronts = false;
		list_add(&pdd->per_device_list, &p->per_device_data);
	}

	return pdd;
}

/*
 * Direct the IOMMU to bind the process (specifically the pasid->mm)
 * to the device.
 * Unbinding occurs when the process dies or the device is removed.
 *
 * Assumes that the process lock is held.
 */
struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
							struct kfd_process *p)
{
	struct kfd_process_device *pdd;
	int err;

	pdd = kfd_get_process_device_data(dev, p);
	if (!pdd) {
		pr_err("Process device data doesn't exist\n");
		return ERR_PTR(-ENOMEM);
	}

	if (pdd->bound)
		return pdd;

	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
	if (err < 0)
		return ERR_PTR(err);

	pdd->bound = true;

	return pdd;
}

void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
{
	struct kfd_process *p;
	struct kfd_process_device *pdd;
	int idx, i;

	BUG_ON(dev == NULL);

	idx = srcu_read_lock(&kfd_processes_srcu);

	/*
	 * Look for the process that matches the pasid. If there is no such
	 * process, we either released it in amdkfd's own notifier, or there
	 * is a bug. Unfortunately, there is no way to tell...
	 */
	hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
		if (p->pasid == pasid) {

			srcu_read_unlock(&kfd_processes_srcu, idx);

			pr_debug("Unbinding process %d from IOMMU\n", pasid);

			mutex_lock(&p->mutex);

			if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
				kfd_dbgmgr_destroy(dev->dbgmgr);

			pqm_uninit(&p->pqm);

			pdd = kfd_get_process_device_data(dev, p);

			if (!pdd) {
				mutex_unlock(&p->mutex);
				return;
			}

			if (pdd->reset_wavefronts) {
				dbgdev_wave_reset_wavefronts(pdd->dev, p);
				pdd->reset_wavefronts = false;
			}

			/*
			 * Just mark pdd as unbound, because we still need it
			 * to call amd_iommu_unbind_pasid() in when the
			 * process exits.
			 * We don't call amd_iommu_unbind_pasid() here
			 * because the IOMMU called us.
			 */
			pdd->bound = false;

			mutex_unlock(&p->mutex);

			return;
		}

	srcu_read_unlock(&kfd_processes_srcu, idx);
}

struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
{
	return list_first_entry(&p->per_device_data,
				struct kfd_process_device,
				per_device_list);
}

struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
						struct kfd_process_device *pdd)
{
	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
		return NULL;
	return list_next_entry(pdd, per_device_list);
}

bool kfd_has_process_device_data(struct kfd_process *p)
{
	return !(list_empty(&p->per_device_data));
}

/* This returns with process->mutex locked. */
struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
{
	struct kfd_process *p;
	unsigned int temp;

	int idx = srcu_read_lock(&kfd_processes_srcu);

	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
		if (p->pasid == pasid) {
			mutex_lock(&p->mutex);
			break;
		}
	}

	srcu_read_unlock(&kfd_processes_srcu, idx);

	return p;
}
Commit	Line	Data
19f6d2a6 OG	1	/*
	2	* Copyright 2014 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*/
	22
	23	#include <linux/mutex.h>
	24	#include <linux/log2.h>
	25	#include <linux/sched.h>
	26	#include <linux/slab.h>
b17f068a	27	#include <linux/amd-iommu.h>
19f6d2a6	28	#include <linux/notifier.h>
dd59239a AS	29	#include <linux/compat.h>
dd59239a AS	30
19f6d2a6 OG	31	struct mm_struct;
	32
	33	#include "kfd_priv.h"
c3447e81	34	#include "kfd_dbgmgr.h"
19f6d2a6 OG	35
	36	/*
	37	* Initial size for the array of queues.
	38	* The allocated size is doubled each time
	39	* it is exceeded up to MAX_PROCESS_QUEUES.
	40	*/
	41	#define INITIAL_QUEUE_ARRAY_SIZE 16
	42
	43	/*
	44	* List of struct kfd_process (field kfd_process).
	45	* Unique/indexed by mm_struct*
	46	*/
	47	#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
	48	static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
	49	static DEFINE_MUTEX(kfd_processes_mutex);
	50
	51	DEFINE_STATIC_SRCU(kfd_processes_srcu);
	52
	53	static struct workqueue_struct *kfd_process_wq;
	54
	55	struct kfd_process_release_work {
	56	struct work_struct kfd_work;
	57	struct kfd_process *p;
	58	};
	59
	60	static struct kfd_process find_process(const struct task_struct thread);
	61	static struct kfd_process create_process(const struct task_struct thread);
	62
	63	void kfd_process_create_wq(void)
	64	{
	65	if (!kfd_process_wq)
	66	kfd_process_wq = create_workqueue("kfd_process_wq");
	67	}
	68
	69	void kfd_process_destroy_wq(void)
	70	{
	71	if (kfd_process_wq) {
	72	flush_workqueue(kfd_process_wq);
	73	destroy_workqueue(kfd_process_wq);
	74	kfd_process_wq = NULL;
	75	}
	76	}
	77
	78	struct kfd_process kfd_create_process(const struct task_struct thread)
	79	{
	80	struct kfd_process *process;
	81
	82	BUG_ON(!kfd_process_wq);
	83
	84	if (thread->mm == NULL)
	85	return ERR_PTR(-EINVAL);
	86
	87	/* Only the pthreads threading model is supported. */
	88	if (thread->group_leader->mm != thread->mm)
	89	return ERR_PTR(-EINVAL);
	90
	91	/* Take mmap_sem because we call __mmu_notifier_register inside */
	92	down_write(&thread->mm->mmap_sem);
	93
	94	/*
	95	* take kfd processes mutex before starting of process creation
	96	* so there won't be a case where two threads of the same process
	97	* create two kfd_process structures
	98	*/
99	mutex_lock(&kfd_processes_mutex);
100
101	/* A prior open of /dev/kfd could have already created the process. */
102	process = find_process(thread);
103	if (process)
104	pr_debug("kfd: process already found\n");
105
106	if (!process)
107	process = create_process(thread);
108
109	mutex_unlock(&kfd_processes_mutex);
110
111	up_write(&thread->mm->mmap_sem);
112
113	return process;
114	}
115
116	struct kfd_process kfd_get_process(const struct task_struct thread)
117	{
118	struct kfd_process *process;
119
120	if (thread->mm == NULL)
121	return ERR_PTR(-EINVAL);
122
123	/* Only the pthreads threading model is supported. */
124	if (thread->group_leader->mm != thread->mm)
125	return ERR_PTR(-EINVAL);
126
127	process = find_process(thread);
128
129	return process;
130	}
131
132	static struct kfd_process find_process_by_mm(const struct mm_struct mm)
133	{
134	struct kfd_process *process;
135
136	hash_for_each_possible_rcu(kfd_processes_table, process,
137	kfd_processes, (uintptr_t)mm)
138	if (process->mm == mm)
139	return process;
140
141	return NULL;
142	}
143
144	static struct kfd_process find_process(const struct task_struct thread)
145	{
146	struct kfd_process *p;
147	int idx;
148
149	idx = srcu_read_lock(&kfd_processes_srcu);
150	p = find_process_by_mm(thread->mm);
151	srcu_read_unlock(&kfd_processes_srcu, idx);
152
153	return p;
154	}
155
156	static void kfd_process_wq_release(struct work_struct *work)
157	{
158	struct kfd_process_release_work *my_work;
159	struct kfd_process_device pdd, temp;
160	struct kfd_process *p;
161
162	my_work = (struct kfd_process_release_work *) work;
163
164	p = my_work->p;
165
94a1ee09 OG	166	pr_debug("Releasing process (pasid %d) in workqueue\n",
	167	p->pasid);
	168
19f6d2a6 OG	169	mutex_lock(&p->mutex);
	170
	171	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
	172	per_device_list) {
94a1ee09 OG	173	pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
	174	pdd->dev->id, p->pasid);
	175
a82918f1	176	if (pdd->reset_wavefronts)
c3447e81 BG	177	dbgdev_wave_reset_wavefronts(pdd->dev, p);
c3447e81 BG	178
b17f068a	179	amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
19f6d2a6 OG	180	list_del(&pdd->per_device_list);
	181
	182	kfree(pdd);
	183	}
	184
f3a39818 AL	185	kfd_event_free_process(p);
f3a39818 AL	186
19f6d2a6 OG	187	kfd_pasid_free(p->pasid);
	188
	189	mutex_unlock(&p->mutex);
	190
	191	mutex_destroy(&p->mutex);
	192
	193	kfree(p->queues);
	194
	195	kfree(p);
	196
642f0f2a	197	kfree(work);
19f6d2a6 OG	198	}
	199
	200	static void kfd_process_destroy_delayed(struct rcu_head *rcu)
	201	{
	202	struct kfd_process_release_work *work;
	203	struct kfd_process *p;
	204
	205	BUG_ON(!kfd_process_wq);
	206
	207	p = container_of(rcu, struct kfd_process, rcu);
	208	BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
	209
	210	mmdrop(p->mm);
	211
1549fcd1	212	work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
19f6d2a6 OG	213
	214	if (work) {
	215	INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
	216	work->p = p;
	217	queue_work(kfd_process_wq, (struct work_struct *) work);
	218	}
	219	}
	220
	221	static void kfd_process_notifier_release(struct mmu_notifier *mn,
	222	struct mm_struct *mm)
	223	{
	224	struct kfd_process *p;
a82918f1	225	struct kfd_process_device *pdd = NULL;
19f6d2a6 OG	226
	227	/*
	228	* The kfd_process structure can not be free because the
	229	* mmu_notifier srcu is read locked
	230	*/
	231	p = container_of(mn, struct kfd_process, mmu_notifier);
	232	BUG_ON(p->mm != mm);
	233
	234	mutex_lock(&kfd_processes_mutex);
	235	hash_del_rcu(&p->kfd_processes);
	236	mutex_unlock(&kfd_processes_mutex);
	237	synchronize_srcu(&kfd_processes_srcu);
	238
45102048 BG	239	mutex_lock(&p->mutex);
	240
	241	/* In case our notifier is called before IOMMU notifier */
	242	pqm_uninit(&p->pqm);
	243
a82918f1	244	/* Iterate over all process device data structure and check
bc4755a4 OG	245	* if we should delete debug managers and reset all wavefronts
	246	*/
	247	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
	248	if ((pdd->dev->dbgmgr) &&
	249	(pdd->dev->dbgmgr->pasid == p->pasid))
	250	kfd_dbgmgr_destroy(pdd->dev->dbgmgr);
	251
a82918f1 BG	252	if (pdd->reset_wavefronts) {
	253	pr_warn("amdkfd: Resetting all wave fronts\n");
	254	dbgdev_wave_reset_wavefronts(pdd->dev, p);
	255	pdd->reset_wavefronts = false;
	256	}
bc4755a4	257	}
a82918f1	258
45102048 BG	259	mutex_unlock(&p->mutex);
45102048 BG	260
19f6d2a6 OG	261	/*
	262	* Because we drop mm_count inside kfd_process_destroy_delayed
	263	* and because the mmu_notifier_unregister function also drop
	264	* mm_count we need to take an extra count here.
	265	*/
	266	atomic_inc(&p->mm->mm_count);
	267	mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
	268	mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
	269	}
	270
	271	static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
	272	.release = kfd_process_notifier_release,
	273	};
	274
	275	static struct kfd_process create_process(const struct task_struct thread)
	276	{
	277	struct kfd_process *process;
	278	int err = -ENOMEM;
	279
	280	process = kzalloc(sizeof(*process), GFP_KERNEL);
	281
	282	if (!process)
	283	goto err_alloc_process;
	284
	285	process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
	286	sizeof(process->queues[0]), GFP_KERNEL);
	287	if (!process->queues)
	288	goto err_alloc_queues;
	289
	290	process->pasid = kfd_pasid_alloc();
	291	if (process->pasid == 0)
	292	goto err_alloc_pasid;
	293
	294	mutex_init(&process->mutex);
	295
	296	process->mm = thread->mm;
	297
	298	/* register notifier */
	299	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
	300	err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
	301	if (err)
	302	goto err_mmu_notifier;
	303
	304	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
	305	(uintptr_t)process->mm);
	306
	307	process->lead_thread = thread->group_leader;
	308
	309	process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
	310
	311	INIT_LIST_HEAD(&process->per_device_data);
	312
f3a39818 AL	313	kfd_event_init_process(process);
f3a39818 AL	314
45102048 BG	315	err = pqm_init(&process->pqm, process);
	316	if (err != 0)
	317	goto err_process_pqm_init;
	318
dd59239a	319	/* init process apertures*/
10f1685f	320	process->is_32bit_user_mode = in_compat_syscall();
dd59239a AS	321	if (kfd_init_apertures(process) != 0)
	322	goto err_init_apretures;
	323
19f6d2a6 OG	324	return process;
19f6d2a6 OG	325
dd59239a AS	326	err_init_apretures:
dd59239a AS	327	pqm_uninit(&process->pqm);
45102048 BG	328	err_process_pqm_init:
	329	hash_del_rcu(&process->kfd_processes);
	330	synchronize_rcu();
	331	mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
19f6d2a6 OG	332	err_mmu_notifier:
	333	kfd_pasid_free(process->pasid);
	334	err_alloc_pasid:
	335	kfree(process->queues);
	336	err_alloc_queues:
	337	kfree(process);
	338	err_alloc_process:
	339	return ERR_PTR(err);
	340	}
	341
	342	struct kfd_process_device kfd_get_process_device_data(struct kfd_dev dev,
093c7d8c	343	struct kfd_process *p)
19f6d2a6 OG	344	{
	345	struct kfd_process_device *pdd = NULL;
	346
	347	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
	348	if (pdd->dev == dev)
093c7d8c AS	349	break;
	350
	351	return pdd;
	352	}
	353
	354	struct kfd_process_device kfd_create_process_device_data(struct kfd_dev dev,
	355	struct kfd_process *p)
	356	{
	357	struct kfd_process_device *pdd = NULL;
	358
	359	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
	360	if (pdd != NULL) {
	361	pdd->dev = dev;
	362	INIT_LIST_HEAD(&pdd->qpd.queues_list);
	363	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
	364	pdd->qpd.dqm = dev->dqm;
a82918f1	365	pdd->reset_wavefronts = false;
093c7d8c	366	list_add(&pdd->per_device_list, &p->per_device_data);
19f6d2a6 OG	367	}
	368
	369	return pdd;
	370	}
	371
	372	/*
	373	* Direct the IOMMU to bind the process (specifically the pasid->mm)
	374	* to the device.
	375	* Unbinding occurs when the process dies or the device is removed.
	376	*
	377	* Assumes that the process lock is held.
	378	*/
	379	struct kfd_process_device kfd_bind_process_to_device(struct kfd_dev dev,
	380	struct kfd_process *p)
	381	{
093c7d8c	382	struct kfd_process_device *pdd;
b17f068a	383	int err;
19f6d2a6	384
093c7d8c AS	385	pdd = kfd_get_process_device_data(dev, p);
	386	if (!pdd) {
	387	pr_err("Process device data doesn't exist\n");
19f6d2a6	388	return ERR_PTR(-ENOMEM);
093c7d8c	389	}
19f6d2a6 OG	390
	391	if (pdd->bound)
	392	return pdd;
	393
b17f068a OG	394	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
	395	if (err < 0)
	396	return ERR_PTR(err);
	397
19f6d2a6 OG	398	pdd->bound = true;
	399
	400	return pdd;
	401	}
	402
	403	void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
	404	{
	405	struct kfd_process *p;
	406	struct kfd_process_device *pdd;
	407	int idx, i;
	408
	409	BUG_ON(dev == NULL);
	410
	411	idx = srcu_read_lock(&kfd_processes_srcu);
	412
121b78e6 OG	413	/*
	414	* Look for the process that matches the pasid. If there is no such
	415	* process, we either released it in amdkfd's own notifier, or there
	416	* is a bug. Unfortunately, there is no way to tell...
	417	*/
19f6d2a6	418	hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
121b78e6	419	if (p->pasid == pasid) {
19f6d2a6	420
121b78e6	421	srcu_read_unlock(&kfd_processes_srcu, idx);
19f6d2a6	422
121b78e6	423	pr_debug("Unbinding process %d from IOMMU\n", pasid);
19f6d2a6	424
121b78e6	425	mutex_lock(&p->mutex);
19f6d2a6	426
121b78e6 OG	427	if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
121b78e6 OG	428	kfd_dbgmgr_destroy(dev->dbgmgr);
c3447e81	429
121b78e6	430	pqm_uninit(&p->pqm);
45102048	431
121b78e6	432	pdd = kfd_get_process_device_data(dev, p);
a0f67441	433
121b78e6 OG	434	if (!pdd) {
	435	mutex_unlock(&p->mutex);
	436	return;
	437	}
a0f67441	438
121b78e6 OG	439	if (pdd->reset_wavefronts) {
	440	dbgdev_wave_reset_wavefronts(pdd->dev, p);
	441	pdd->reset_wavefronts = false;
	442	}
19f6d2a6	443
121b78e6 OG	444	/*
	445	* Just mark pdd as unbound, because we still need it
	446	* to call amd_iommu_unbind_pasid() in when the
	447	* process exits.
	448	* We don't call amd_iommu_unbind_pasid() here
	449	* because the IOMMU called us.
	450	*/
	451	pdd->bound = false;
19f6d2a6	452
121b78e6 OG	453	mutex_unlock(&p->mutex);
	454
	455	return;
	456	}
	457
	458	srcu_read_unlock(&kfd_processes_srcu, idx);
19f6d2a6 OG	459	}
	460
	461	struct kfd_process_device kfd_get_first_process_device_data(struct kfd_process p)
	462	{
	463	return list_first_entry(&p->per_device_data,
	464	struct kfd_process_device,
	465	per_device_list);
	466	}
	467
	468	struct kfd_process_device kfd_get_next_process_device_data(struct kfd_process p,
	469	struct kfd_process_device *pdd)
	470	{
	471	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
	472	return NULL;
	473	return list_next_entry(pdd, per_device_list);
	474	}
	475
	476	bool kfd_has_process_device_data(struct kfd_process *p)
	477	{
	478	return !(list_empty(&p->per_device_data));
	479	}
f3a39818 AL	480
	481	/* This returns with process->mutex locked. */
	482	struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
	483	{
	484	struct kfd_process *p;
	485	unsigned int temp;
	486
	487	int idx = srcu_read_lock(&kfd_processes_srcu);
	488
	489	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
	490	if (p->pasid == pasid) {
	491	mutex_lock(&p->mutex);
	492	break;
	493	}
	494	}
	495
	496	srcu_read_unlock(&kfd_processes_srcu, idx);
	497
	498	return p;
	499	}