[deliverable/linux.git] / arch / powerpc / kvm / book3s_64_vio.c

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 *
 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 * Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com>
 * Copyright 2016 Alexey Kardashevskiy, IBM Corporation <aik@au1.ibm.com>
 */

#include <linux/types.h>
#include <linux/string.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/highmem.h>
#include <linux/gfp.h>
#include <linux/slab.h>
#include <linux/hugetlb.h>
#include <linux/list.h>
#include <linux/anon_inodes.h>

#include <asm/tlbflush.h>
#include <asm/kvm_ppc.h>
#include <asm/kvm_book3s.h>
#include <asm/book3s/64/mmu-hash.h>
#include <asm/hvcall.h>
#include <asm/synch.h>
#include <asm/ppc-opcode.h>
#include <asm/kvm_host.h>
#include <asm/udbg.h>
#include <asm/iommu.h>
#include <asm/tce.h>

static unsigned long kvmppc_tce_pages(unsigned long iommu_pages)
{
	return ALIGN(iommu_pages * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
}

static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
{
	unsigned long stt_bytes = sizeof(struct kvmppc_spapr_tce_table) +
			(tce_pages * sizeof(struct page *));

	return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
}

static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
{
	long ret = 0;

	if (!current || !current->mm)
		return ret; /* process exited */

	down_write(&current->mm->mmap_sem);

	if (inc) {
		unsigned long locked, lock_limit;

		locked = current->mm->locked_vm + stt_pages;
		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
			ret = -ENOMEM;
		else
			current->mm->locked_vm += stt_pages;
	} else {
		if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
			stt_pages = current->mm->locked_vm;

		current->mm->locked_vm -= stt_pages;
	}

	pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
			inc ? '+' : '-',
			stt_pages << PAGE_SHIFT,
			current->mm->locked_vm << PAGE_SHIFT,
			rlimit(RLIMIT_MEMLOCK),
			ret ? " - exceeded" : "");

	up_write(&current->mm->mmap_sem);

	return ret;
}

static void release_spapr_tce_table(struct rcu_head *head)
{
	struct kvmppc_spapr_tce_table *stt = container_of(head,
			struct kvmppc_spapr_tce_table, rcu);
	unsigned long i, npages = kvmppc_tce_pages(stt->size);

	for (i = 0; i < npages; i++)
		__free_page(stt->pages[i]);

	kfree(stt);
}

static int kvm_spapr_tce_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct kvmppc_spapr_tce_table *stt = vma->vm_file->private_data;
	struct page *page;

	if (vmf->pgoff >= kvmppc_tce_pages(stt->size))
		return VM_FAULT_SIGBUS;

	page = stt->pages[vmf->pgoff];
	get_page(page);
	vmf->page = page;
	return 0;
}

static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
	.fault = kvm_spapr_tce_fault,
};

static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_spapr_tce_vm_ops;
	return 0;
}

static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
{
	struct kvmppc_spapr_tce_table *stt = filp->private_data;

	list_del_rcu(&stt->list);

	kvm_put_kvm(stt->kvm);

	kvmppc_account_memlimit(
		kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
	call_rcu(&stt->rcu, release_spapr_tce_table);

	return 0;
}

static const struct file_operations kvm_spapr_tce_fops = {
	.mmap           = kvm_spapr_tce_mmap,
	.release	= kvm_spapr_tce_release,
};

long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
				   struct kvm_create_spapr_tce_64 *args)
{
	struct kvmppc_spapr_tce_table *stt = NULL;
	unsigned long npages, size;
	int ret = -ENOMEM;
	int i;

	if (!args->size)
		return -EINVAL;

	/* Check this LIOBN hasn't been previously allocated */
	list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
		if (stt->liobn == args->liobn)
			return -EBUSY;
	}

	size = args->size;
	npages = kvmppc_tce_pages(size);
	ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
	if (ret) {
		stt = NULL;
		goto fail;
	}

	stt = kzalloc(sizeof(*stt) + npages * sizeof(struct page *),
		      GFP_KERNEL);
	if (!stt)
		goto fail;

	stt->liobn = args->liobn;
	stt->page_shift = args->page_shift;
	stt->offset = args->offset;
	stt->size = size;
	stt->kvm = kvm;

	for (i = 0; i < npages; i++) {
		stt->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!stt->pages[i])
			goto fail;
	}

	kvm_get_kvm(kvm);

	mutex_lock(&kvm->lock);
	list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables);

	mutex_unlock(&kvm->lock);

	return anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
				stt, O_RDWR | O_CLOEXEC);

fail:
	if (stt) {
		for (i = 0; i < npages; i++)
			if (stt->pages[i])
				__free_page(stt->pages[i]);

		kfree(stt);
	}
	return ret;
}

long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
		      unsigned long ioba, unsigned long tce)
{
	struct kvmppc_spapr_tce_table *stt = kvmppc_find_table(vcpu, liobn);
	long ret;

	/* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */
	/* 	    liobn, ioba, tce); */

	if (!stt)
		return H_TOO_HARD;

	ret = kvmppc_ioba_validate(stt, ioba, 1);
	if (ret != H_SUCCESS)
		return ret;

	ret = kvmppc_tce_validate(stt, tce);
	if (ret != H_SUCCESS)
		return ret;

	kvmppc_tce_put(stt, ioba >> stt->page_shift, tce);

	return H_SUCCESS;
}
EXPORT_SYMBOL_GPL(kvmppc_h_put_tce);

long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
		unsigned long liobn, unsigned long ioba,
		unsigned long tce_list, unsigned long npages)
{
	struct kvmppc_spapr_tce_table *stt;
	long i, ret = H_SUCCESS, idx;
	unsigned long entry, ua = 0;
	u64 __user *tces, tce;

	stt = kvmppc_find_table(vcpu, liobn);
	if (!stt)
		return H_TOO_HARD;

	entry = ioba >> stt->page_shift;
	/*
	 * SPAPR spec says that the maximum size of the list is 512 TCEs
	 * so the whole table fits in 4K page
	 */
	if (npages > 512)
		return H_PARAMETER;

	if (tce_list & (SZ_4K - 1))
		return H_PARAMETER;

	ret = kvmppc_ioba_validate(stt, ioba, npages);
	if (ret != H_SUCCESS)
		return ret;

	idx = srcu_read_lock(&vcpu->kvm->srcu);
	if (kvmppc_gpa_to_ua(vcpu->kvm, tce_list, &ua, NULL)) {
		ret = H_TOO_HARD;
		goto unlock_exit;
	}
	tces = (u64 __user *) ua;

	for (i = 0; i < npages; ++i) {
		if (get_user(tce, tces + i)) {
			ret = H_TOO_HARD;
			goto unlock_exit;
		}
		tce = be64_to_cpu(tce);

		ret = kvmppc_tce_validate(stt, tce);
		if (ret != H_SUCCESS)
			goto unlock_exit;

		kvmppc_tce_put(stt, entry + i, tce);
	}

unlock_exit:
	srcu_read_unlock(&vcpu->kvm->srcu, idx);

	return ret;
}
EXPORT_SYMBOL_GPL(kvmppc_h_put_tce_indirect);

long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu,
		unsigned long liobn, unsigned long ioba,
		unsigned long tce_value, unsigned long npages)
{
	struct kvmppc_spapr_tce_table *stt;
	long i, ret;

	stt = kvmppc_find_table(vcpu, liobn);
	if (!stt)
		return H_TOO_HARD;

	ret = kvmppc_ioba_validate(stt, ioba, npages);
	if (ret != H_SUCCESS)
		return ret;

	/* Check permission bits only to allow userspace poison TCE for debug */
	if (tce_value & (TCE_PCI_WRITE | TCE_PCI_READ))
		return H_PARAMETER;

	for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))
		kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value);

	return H_SUCCESS;
}
EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce);
Commit	Line	Data
f31e65e1 BH	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License, version 2, as
	4	* published by the Free Software Foundation.
	5	*
	6	* This program is distributed in the hope that it will be useful,
	7	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	8	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	9	* GNU General Public License for more details.
	10	*
	11	* You should have received a copy of the GNU General Public License
	12	* along with this program; if not, write to the Free Software
	13	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	14	*
	15	* Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
	16	* Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com>
d3695aa4	17	* Copyright 2016 Alexey Kardashevskiy, IBM Corporation <aik@au1.ibm.com>
f31e65e1 BH	18	*/
	19
	20	#include <linux/types.h>
	21	#include <linux/string.h>
	22	#include <linux/kvm.h>
	23	#include <linux/kvm_host.h>
	24	#include <linux/highmem.h>
	25	#include <linux/gfp.h>
	26	#include <linux/slab.h>
	27	#include <linux/hugetlb.h>
	28	#include <linux/list.h>
	29	#include <linux/anon_inodes.h>
	30
	31	#include <asm/tlbflush.h>
	32	#include <asm/kvm_ppc.h>
	33	#include <asm/kvm_book3s.h>
f64e8084	34	#include <asm/book3s/64/mmu-hash.h>
f31e65e1 BH	35	#include <asm/hvcall.h>
	36	#include <asm/synch.h>
	37	#include <asm/ppc-opcode.h>
	38	#include <asm/kvm_host.h>
	39	#include <asm/udbg.h>
462ee11e	40	#include <asm/iommu.h>
d3695aa4	41	#include <asm/tce.h>
f31e65e1	42
fe26e527	43	static unsigned long kvmppc_tce_pages(unsigned long iommu_pages)
f31e65e1	44	{
fe26e527	45	return ALIGN(iommu_pages * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
f31e65e1 BH	46	}
f31e65e1 BH	47
f8626985 AK	48	static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
	49	{
	50	unsigned long stt_bytes = sizeof(struct kvmppc_spapr_tce_table) +
	51	(tce_pages * sizeof(struct page *));
	52
	53	return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
	54	}
	55
	56	static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
	57	{
	58	long ret = 0;
	59
	60	if (!current \|\| !current->mm)
	61	return ret; /* process exited */
	62
	63	down_write(&current->mm->mmap_sem);
	64
	65	if (inc) {
	66	unsigned long locked, lock_limit;
	67
	68	locked = current->mm->locked_vm + stt_pages;
	69	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	70	if (locked > lock_limit && !capable(CAP_IPC_LOCK))
	71	ret = -ENOMEM;
	72	else
	73	current->mm->locked_vm += stt_pages;
	74	} else {
	75	if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
	76	stt_pages = current->mm->locked_vm;
	77
	78	current->mm->locked_vm -= stt_pages;
	79	}
	80
	81	pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
	82	inc ? '+' : '-',
	83	stt_pages << PAGE_SHIFT,
	84	current->mm->locked_vm << PAGE_SHIFT,
	85	rlimit(RLIMIT_MEMLOCK),
	86	ret ? " - exceeded" : "");
	87
	88	up_write(&current->mm->mmap_sem);
	89
	90	return ret;
	91	}
	92
366baf28	93	static void release_spapr_tce_table(struct rcu_head *head)
f31e65e1	94	{
366baf28 AK	95	struct kvmppc_spapr_tce_table *stt = container_of(head,
366baf28 AK	96	struct kvmppc_spapr_tce_table, rcu);
fe26e527	97	unsigned long i, npages = kvmppc_tce_pages(stt->size);
f31e65e1	98
f8626985	99	for (i = 0; i < npages; i++)
f31e65e1	100	__free_page(stt->pages[i]);
f31e65e1	101
366baf28	102	kfree(stt);
f31e65e1 BH	103	}
	104
	105	static int kvm_spapr_tce_fault(struct vm_area_struct vma, struct vm_fault vmf)
	106	{
	107	struct kvmppc_spapr_tce_table *stt = vma->vm_file->private_data;
	108	struct page *page;
	109
fe26e527	110	if (vmf->pgoff >= kvmppc_tce_pages(stt->size))
f31e65e1 BH	111	return VM_FAULT_SIGBUS;
	112
	113	page = stt->pages[vmf->pgoff];
	114	get_page(page);
	115	vmf->page = page;
	116	return 0;
	117	}
	118
	119	static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
	120	.fault = kvm_spapr_tce_fault,
	121	};
	122
	123	static int kvm_spapr_tce_mmap(struct file file, struct vm_area_struct vma)
	124	{
	125	vma->vm_ops = &kvm_spapr_tce_vm_ops;
	126	return 0;
	127	}
	128
	129	static int kvm_spapr_tce_release(struct inode inode, struct file filp)
	130	{
	131	struct kvmppc_spapr_tce_table *stt = filp->private_data;
	132
366baf28 AK	133	list_del_rcu(&stt->list);
	134
	135	kvm_put_kvm(stt->kvm);
	136
f8626985	137	kvmppc_account_memlimit(
fe26e527	138	kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
366baf28 AK	139	call_rcu(&stt->rcu, release_spapr_tce_table);
366baf28 AK	140
f31e65e1 BH	141	return 0;
	142	}
	143
75ef9de1	144	static const struct file_operations kvm_spapr_tce_fops = {
f31e65e1 BH	145	.mmap = kvm_spapr_tce_mmap,
	146	.release = kvm_spapr_tce_release,
	147	};
	148
	149	long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
58ded420	150	struct kvm_create_spapr_tce_64 *args)
f31e65e1 BH	151	{
f31e65e1 BH	152	struct kvmppc_spapr_tce_table *stt = NULL;
fe26e527	153	unsigned long npages, size;
f31e65e1 BH	154	int ret = -ENOMEM;
	155	int i;
	156
58ded420 AK	157	if (!args->size)
	158	return -EINVAL;
	159
f31e65e1 BH	160	/* Check this LIOBN hasn't been previously allocated */
	161	list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
	162	if (stt->liobn == args->liobn)
	163	return -EBUSY;
	164	}
	165
58ded420	166	size = args->size;
fe26e527	167	npages = kvmppc_tce_pages(size);
f8626985 AK	168	ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
	169	if (ret) {
	170	stt = NULL;
	171	goto fail;
	172	}
f31e65e1 BH	173
	174	stt = kzalloc(sizeof(stt) + npages sizeof(struct page *),
	175	GFP_KERNEL);
	176	if (!stt)
	177	goto fail;
	178
	179	stt->liobn = args->liobn;
58ded420 AK	180	stt->page_shift = args->page_shift;
58ded420 AK	181	stt->offset = args->offset;
fe26e527	182	stt->size = size;
f31e65e1 BH	183	stt->kvm = kvm;
	184
	185	for (i = 0; i < npages; i++) {
	186	stt->pages[i] = alloc_page(GFP_KERNEL \| __GFP_ZERO);
	187	if (!stt->pages[i])
	188	goto fail;
	189	}
	190
	191	kvm_get_kvm(kvm);
	192
	193	mutex_lock(&kvm->lock);
366baf28	194	list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables);
f31e65e1 BH	195
	196	mutex_unlock(&kvm->lock);
	197
	198	return anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
2f84d5ea	199	stt, O_RDWR \| O_CLOEXEC);
f31e65e1 BH	200
	201	fail:
	202	if (stt) {
	203	for (i = 0; i < npages; i++)
	204	if (stt->pages[i])
	205	__free_page(stt->pages[i]);
	206
	207	kfree(stt);
	208	}
	209	return ret;
	210	}
d3695aa4	211
31217db7 AK	212	long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
	213	unsigned long ioba, unsigned long tce)
	214	{
	215	struct kvmppc_spapr_tce_table *stt = kvmppc_find_table(vcpu, liobn);
	216	long ret;
	217
	218	/* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */
	219	/* liobn, ioba, tce); */
	220
	221	if (!stt)
	222	return H_TOO_HARD;
	223
	224	ret = kvmppc_ioba_validate(stt, ioba, 1);
	225	if (ret != H_SUCCESS)
	226	return ret;
	227
	228	ret = kvmppc_tce_validate(stt, tce);
	229	if (ret != H_SUCCESS)
	230	return ret;
	231
	232	kvmppc_tce_put(stt, ioba >> stt->page_shift, tce);
	233
	234	return H_SUCCESS;
	235	}
	236	EXPORT_SYMBOL_GPL(kvmppc_h_put_tce);
	237
d3695aa4 AK	238	long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
	239	unsigned long liobn, unsigned long ioba,
	240	unsigned long tce_list, unsigned long npages)
	241	{
	242	struct kvmppc_spapr_tce_table *stt;
	243	long i, ret = H_SUCCESS, idx;
	244	unsigned long entry, ua = 0;
	245	u64 __user *tces, tce;
	246
	247	stt = kvmppc_find_table(vcpu, liobn);
	248	if (!stt)
	249	return H_TOO_HARD;
	250
fe26e527	251	entry = ioba >> stt->page_shift;
d3695aa4 AK	252	/*
	253	* SPAPR spec says that the maximum size of the list is 512 TCEs
	254	* so the whole table fits in 4K page
	255	*/
	256	if (npages > 512)
	257	return H_PARAMETER;
	258
	259	if (tce_list & (SZ_4K - 1))
	260	return H_PARAMETER;
	261
	262	ret = kvmppc_ioba_validate(stt, ioba, npages);
	263	if (ret != H_SUCCESS)
	264	return ret;
	265
	266	idx = srcu_read_lock(&vcpu->kvm->srcu);
	267	if (kvmppc_gpa_to_ua(vcpu->kvm, tce_list, &ua, NULL)) {
	268	ret = H_TOO_HARD;
	269	goto unlock_exit;
	270	}
	271	tces = (u64 __user *) ua;
	272
	273	for (i = 0; i < npages; ++i) {
	274	if (get_user(tce, tces + i)) {
	275	ret = H_TOO_HARD;
	276	goto unlock_exit;
	277	}
	278	tce = be64_to_cpu(tce);
	279
	280	ret = kvmppc_tce_validate(stt, tce);
	281	if (ret != H_SUCCESS)
	282	goto unlock_exit;
	283
	284	kvmppc_tce_put(stt, entry + i, tce);
	285	}
	286
	287	unlock_exit:
	288	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	289
	290	return ret;
	291	}
	292	EXPORT_SYMBOL_GPL(kvmppc_h_put_tce_indirect);
31217db7 AK	293
	294	long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu,
	295	unsigned long liobn, unsigned long ioba,
	296	unsigned long tce_value, unsigned long npages)
	297	{
	298	struct kvmppc_spapr_tce_table *stt;
	299	long i, ret;
	300
	301	stt = kvmppc_find_table(vcpu, liobn);
	302	if (!stt)
	303	return H_TOO_HARD;
	304
	305	ret = kvmppc_ioba_validate(stt, ioba, npages);
	306	if (ret != H_SUCCESS)
	307	return ret;
	308
	309	/* Check permission bits only to allow userspace poison TCE for debug */
	310	if (tce_value & (TCE_PCI_WRITE \| TCE_PCI_READ))
	311	return H_PARAMETER;
	312
	313	for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))
	314	kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value);
	315
	316	return H_SUCCESS;
	317	}
	318	EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce);