[deliverable/linux.git] / arch / um / kernel / tlb.c

/*
 * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
 * Licensed under the GPL
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <as-layout.h>
#include <mem_user.h>
#include <os.h>
#include <skas.h>
#include <kern_util.h>

struct host_vm_change {
	struct host_vm_op {
		enum { NONE, MMAP, MUNMAP, MPROTECT } type;
		union {
			struct {
				unsigned long addr;
				unsigned long len;
				unsigned int prot;
				int fd;
				__u64 offset;
			} mmap;
			struct {
				unsigned long addr;
				unsigned long len;
			} munmap;
			struct {
				unsigned long addr;
				unsigned long len;
				unsigned int prot;
			} mprotect;
		} u;
	} ops[1];
	int index;
	struct mm_id *id;
	void *data;
	int force;
};

#define INIT_HVC(mm, force) \
	((struct host_vm_change) \
	 { .ops		= { { .type = NONE } },	\
	   .id		= &mm->context.id, \
       	   .data	= NULL, \
	   .index	= 0, \
	   .force	= force })

static int do_ops(struct host_vm_change *hvc, int end,
		  int finished)
{
	struct host_vm_op *op;
	int i, ret = 0;

	for (i = 0; i < end && !ret; i++) {
		op = &hvc->ops[i];
		switch (op->type) {
		case MMAP:
			ret = map(hvc->id, op->u.mmap.addr, op->u.mmap.len,
				  op->u.mmap.prot, op->u.mmap.fd,
				  op->u.mmap.offset, finished, &hvc->data);
			break;
		case MUNMAP:
			ret = unmap(hvc->id, op->u.munmap.addr,
				    op->u.munmap.len, finished, &hvc->data);
			break;
		case MPROTECT:
			ret = protect(hvc->id, op->u.mprotect.addr,
				      op->u.mprotect.len, op->u.mprotect.prot,
				      finished, &hvc->data);
			break;
		default:
			printk(KERN_ERR "Unknown op type %d in do_ops\n",
			       op->type);
			BUG();
			break;
		}
	}

	return ret;
}

static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
		    unsigned int prot, struct host_vm_change *hvc)
{
	__u64 offset;
	struct host_vm_op *last;
	int fd, ret = 0;

	fd = phys_mapping(phys, &offset);
	if (hvc->index != 0) {
		last = &hvc->ops[hvc->index - 1];
		if ((last->type == MMAP) &&
		   (last->u.mmap.addr + last->u.mmap.len == virt) &&
		   (last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) &&
		   (last->u.mmap.offset + last->u.mmap.len == offset)) {
			last->u.mmap.len += len;
			return 0;
		}
	}

	if (hvc->index == ARRAY_SIZE(hvc->ops)) {
		ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
		hvc->index = 0;
	}

	hvc->ops[hvc->index++] = ((struct host_vm_op)
				  { .type	= MMAP,
				    .u = { .mmap = { .addr	= virt,
						     .len	= len,
						     .prot	= prot,
						     .fd	= fd,
						     .offset	= offset }
			   } });
	return ret;
}

static int add_munmap(unsigned long addr, unsigned long len,
		      struct host_vm_change *hvc)
{
	struct host_vm_op *last;
	int ret = 0;

	if ((addr >= STUB_START) && (addr < STUB_END))
		return -EINVAL;

	if (hvc->index != 0) {
		last = &hvc->ops[hvc->index - 1];
		if ((last->type == MUNMAP) &&
		   (last->u.munmap.addr + last->u.mmap.len == addr)) {
			last->u.munmap.len += len;
			return 0;
		}
	}

	if (hvc->index == ARRAY_SIZE(hvc->ops)) {
		ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
		hvc->index = 0;
	}

	hvc->ops[hvc->index++] = ((struct host_vm_op)
				  { .type	= MUNMAP,
			     	    .u = { .munmap = { .addr	= addr,
						       .len	= len } } });
	return ret;
}

static int add_mprotect(unsigned long addr, unsigned long len,
			unsigned int prot, struct host_vm_change *hvc)
{
	struct host_vm_op *last;
	int ret = 0;

	if (hvc->index != 0) {
		last = &hvc->ops[hvc->index - 1];
		if ((last->type == MPROTECT) &&
		   (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
		   (last->u.mprotect.prot == prot)) {
			last->u.mprotect.len += len;
			return 0;
		}
	}

	if (hvc->index == ARRAY_SIZE(hvc->ops)) {
		ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
		hvc->index = 0;
	}

	hvc->ops[hvc->index++] = ((struct host_vm_op)
				  { .type	= MPROTECT,
			     	    .u = { .mprotect = { .addr	= addr,
							 .len	= len,
							 .prot	= prot } } });
	return ret;
}

#define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))

static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
				   unsigned long end,
				   struct host_vm_change *hvc)
{
	pte_t *pte;
	int r, w, x, prot, ret = 0;

	pte = pte_offset_kernel(pmd, addr);
	do {
		if ((addr >= STUB_START) && (addr < STUB_END))
			continue;

		r = pte_read(*pte);
		w = pte_write(*pte);
		x = pte_exec(*pte);
		if (!pte_young(*pte)) {
			r = 0;
			w = 0;
		} else if (!pte_dirty(*pte))
			w = 0;

		prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
			(x ? UM_PROT_EXEC : 0));
		if (hvc->force || pte_newpage(*pte)) {
			if (pte_present(*pte))
				ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
					       PAGE_SIZE, prot, hvc);
			else
				ret = add_munmap(addr, PAGE_SIZE, hvc);
		} else if (pte_newprot(*pte))
			ret = add_mprotect(addr, PAGE_SIZE, prot, hvc);
		*pte = pte_mkuptodate(*pte);
	} while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret));
	return ret;
}

static inline int update_pmd_range(pud_t *pud, unsigned long addr,
				   unsigned long end,
				   struct host_vm_change *hvc)
{
	pmd_t *pmd;
	unsigned long next;
	int ret = 0;

	pmd = pmd_offset(pud, addr);
	do {
		next = pmd_addr_end(addr, end);
		if (!pmd_present(*pmd)) {
			if (hvc->force || pmd_newpage(*pmd)) {
				ret = add_munmap(addr, next - addr, hvc);
				pmd_mkuptodate(*pmd);
			}
		}
		else ret = update_pte_range(pmd, addr, next, hvc);
	} while (pmd++, addr = next, ((addr < end) && !ret));
	return ret;
}

static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
				   unsigned long end,
				   struct host_vm_change *hvc)
{
	pud_t *pud;
	unsigned long next;
	int ret = 0;

	pud = pud_offset(pgd, addr);
	do {
		next = pud_addr_end(addr, end);
		if (!pud_present(*pud)) {
			if (hvc->force || pud_newpage(*pud)) {
				ret = add_munmap(addr, next - addr, hvc);
				pud_mkuptodate(*pud);
			}
		}
		else ret = update_pmd_range(pud, addr, next, hvc);
	} while (pud++, addr = next, ((addr < end) && !ret));
	return ret;
}

void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
		      unsigned long end_addr, int force)
{
	pgd_t *pgd;
	struct host_vm_change hvc;
	unsigned long addr = start_addr, next;
	int ret = 0;

	hvc = INIT_HVC(mm, force);
	pgd = pgd_offset(mm, addr);
	do {
		next = pgd_addr_end(addr, end_addr);
		if (!pgd_present(*pgd)) {
			if (force || pgd_newpage(*pgd)) {
				ret = add_munmap(addr, next - addr, &hvc);
				pgd_mkuptodate(*pgd);
			}
		}
		else ret = update_pud_range(pgd, addr, next, &hvc);
	} while (pgd++, addr = next, ((addr < end_addr) && !ret));

	if (!ret)
		ret = do_ops(&hvc, hvc.index, 1);

	/* This is not an else because ret is modified above */
	if (ret) {
		printk(KERN_ERR "fix_range_common: failed, killing current "
		       "process: %d\n", task_tgid_vnr(current));
		/* We are under mmap_sem, release it such that current can terminate */
		up_write(&current->mm->mmap_sem);
		force_sig(SIGKILL, current);
		do_signal();
	}
}

static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
{
	struct mm_struct *mm;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
	unsigned long addr, last;
	int updated = 0, err;

	mm = &init_mm;
	for (addr = start; addr < end;) {
		pgd = pgd_offset(mm, addr);
		if (!pgd_present(*pgd)) {
			last = ADD_ROUND(addr, PGDIR_SIZE);
			if (last > end)
				last = end;
			if (pgd_newpage(*pgd)) {
				updated = 1;
				err = os_unmap_memory((void *) addr,
						      last - addr);
				if (err < 0)
					panic("munmap failed, errno = %d\n",
					      -err);
			}
			addr = last;
			continue;
		}

		pud = pud_offset(pgd, addr);
		if (!pud_present(*pud)) {
			last = ADD_ROUND(addr, PUD_SIZE);
			if (last > end)
				last = end;
			if (pud_newpage(*pud)) {
				updated = 1;
				err = os_unmap_memory((void *) addr,
						      last - addr);
				if (err < 0)
					panic("munmap failed, errno = %d\n",
					      -err);
			}
			addr = last;
			continue;
		}

		pmd = pmd_offset(pud, addr);
		if (!pmd_present(*pmd)) {
			last = ADD_ROUND(addr, PMD_SIZE);
			if (last > end)
				last = end;
			if (pmd_newpage(*pmd)) {
				updated = 1;
				err = os_unmap_memory((void *) addr,
						      last - addr);
				if (err < 0)
					panic("munmap failed, errno = %d\n",
					      -err);
			}
			addr = last;
			continue;
		}

		pte = pte_offset_kernel(pmd, addr);
		if (!pte_present(*pte) || pte_newpage(*pte)) {
			updated = 1;
			err = os_unmap_memory((void *) addr,
					      PAGE_SIZE);
			if (err < 0)
				panic("munmap failed, errno = %d\n",
				      -err);
			if (pte_present(*pte))
				map_memory(addr,
					   pte_val(*pte) & PAGE_MASK,
					   PAGE_SIZE, 1, 1, 1);
		}
		else if (pte_newprot(*pte)) {
			updated = 1;
			os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
		}
		addr += PAGE_SIZE;
	}
	return updated;
}

void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
	struct mm_struct *mm = vma->vm_mm;
	void *flush = NULL;
	int r, w, x, prot, err = 0;
	struct mm_id *mm_id;

	address &= PAGE_MASK;
	pgd = pgd_offset(mm, address);
	if (!pgd_present(*pgd))
		goto kill;

	pud = pud_offset(pgd, address);
	if (!pud_present(*pud))
		goto kill;

	pmd = pmd_offset(pud, address);
	if (!pmd_present(*pmd))
		goto kill;

	pte = pte_offset_kernel(pmd, address);

	r = pte_read(*pte);
	w = pte_write(*pte);
	x = pte_exec(*pte);
	if (!pte_young(*pte)) {
		r = 0;
		w = 0;
	} else if (!pte_dirty(*pte)) {
		w = 0;
	}

	mm_id = &mm->context.id;
	prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
		(x ? UM_PROT_EXEC : 0));
	if (pte_newpage(*pte)) {
		if (pte_present(*pte)) {
			unsigned long long offset;
			int fd;

			fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset);
			err = map(mm_id, address, PAGE_SIZE, prot, fd, offset,
				  1, &flush);
		}
		else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush);
	}
	else if (pte_newprot(*pte))
		err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush);

	if (err)
		goto kill;

	*pte = pte_mkuptodate(*pte);

	return;

kill:
	printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address);
	force_sig(SIGKILL, current);
}

pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
{
	return pgd_offset(mm, address);
}

pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
{
	return pud_offset(pgd, address);
}

pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
{
	return pmd_offset(pud, address);
}

pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
{
	return pte_offset_kernel(pmd, address);
}

pte_t *addr_pte(struct task_struct *task, unsigned long addr)
{
	pgd_t *pgd = pgd_offset(task->mm, addr);
	pud_t *pud = pud_offset(pgd, addr);
	pmd_t *pmd = pmd_offset(pud, addr);

	return pte_offset_map(pmd, addr);
}

void flush_tlb_all(void)
{
	flush_tlb_mm(current->mm);
}

void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	flush_tlb_kernel_range_common(start, end);
}

void flush_tlb_kernel_vm(void)
{
	flush_tlb_kernel_range_common(start_vm, end_vm);
}

void __flush_tlb_one(unsigned long addr)
{
	flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE);
}

static void fix_range(struct mm_struct *mm, unsigned long start_addr,
		      unsigned long end_addr, int force)
{
	fix_range_common(mm, start_addr, end_addr, force);
}

void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
		     unsigned long end)
{
	if (vma->vm_mm == NULL)
		flush_tlb_kernel_range_common(start, end);
	else fix_range(vma->vm_mm, start, end, 0);
}
EXPORT_SYMBOL(flush_tlb_range);

void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
			unsigned long end)
{
	/*
	 * Don't bother flushing if this address space is about to be
	 * destroyed.
	 */
	if (atomic_read(&mm->mm_users) == 0)
		return;

	fix_range(mm, start, end, 0);
}

void flush_tlb_mm(struct mm_struct *mm)
{
	struct vm_area_struct *vma = mm->mmap;

	while (vma != NULL) {
		fix_range(mm, vma->vm_start, vma->vm_end, 0);
		vma = vma->vm_next;
	}
}

void force_flush_all(void)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma = mm->mmap;

	while (vma != NULL) {
		fix_range(mm, vma->vm_start, vma->vm_end, 1);
		vma = vma->vm_next;
	}
}
Commit	Line	Data
5e1f65a6	1	/*
ba180fd4	2	* Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
1da177e4 LT	3	* Licensed under the GPL
	4	*/
	5
8192ab42	6	#include <linux/mm.h>
73395a00	7	#include <linux/module.h>
8192ab42 JD	8	#include <linux/sched.h>
	9	#include <asm/pgtable.h>
	10	#include <asm/tlbflush.h>
37185b33 AV	11	#include <as-layout.h>
	12	#include <mem_user.h>
	13	#include <os.h>
	14	#include <skas.h>
468f6597	15	#include <kern_util.h>
1da177e4	16
1466abf2 JD	17	struct host_vm_change {
	18	struct host_vm_op {
	19	enum { NONE, MMAP, MUNMAP, MPROTECT } type;
	20	union {
	21	struct {
	22	unsigned long addr;
	23	unsigned long len;
	24	unsigned int prot;
	25	int fd;
	26	__u64 offset;
	27	} mmap;
	28	struct {
	29	unsigned long addr;
	30	unsigned long len;
	31	} munmap;
	32	struct {
	33	unsigned long addr;
	34	unsigned long len;
	35	unsigned int prot;
	36	} mprotect;
	37	} u;
	38	} ops[1];
	39	int index;
	40	struct mm_id *id;
	41	void *data;
	42	int force;
	43	};
	44
	45	#define INIT_HVC(mm, force) \
	46	((struct host_vm_change) \
	47	{ .ops = { { .type = NONE } }, \
	48	.id = &mm->context.id, \
	49	.data = NULL, \
	50	.index = 0, \
	51	.force = force })
	52
	53	static int do_ops(struct host_vm_change *hvc, int end,
	54	int finished)
	55	{
	56	struct host_vm_op *op;
	57	int i, ret = 0;
	58
	59	for (i = 0; i < end && !ret; i++) {
	60	op = &hvc->ops[i];
c5d4bb17	61	switch (op->type) {
1466abf2 JD	62	case MMAP:
	63	ret = map(hvc->id, op->u.mmap.addr, op->u.mmap.len,
	64	op->u.mmap.prot, op->u.mmap.fd,
	65	op->u.mmap.offset, finished, &hvc->data);
	66	break;
	67	case MUNMAP:
	68	ret = unmap(hvc->id, op->u.munmap.addr,
	69	op->u.munmap.len, finished, &hvc->data);
	70	break;
	71	case MPROTECT:
	72	ret = protect(hvc->id, op->u.mprotect.addr,
	73	op->u.mprotect.len, op->u.mprotect.prot,
	74	finished, &hvc->data);
	75	break;
	76	default:
	77	printk(KERN_ERR "Unknown op type %d in do_ops\n",
	78	op->type);
62179d47	79	BUG();
1466abf2 JD	80	break;
	81	}
	82	}
	83
	84	return ret;
	85	}
	86
c5600490	87	static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
1466abf2	88	unsigned int prot, struct host_vm_change *hvc)
c5600490	89	{
5e1f65a6	90	__u64 offset;
c5600490	91	struct host_vm_op *last;
07bf731e	92	int fd, ret = 0;
c5600490 JD	93
c5600490 JD	94	fd = phys_mapping(phys, &offset);
1466abf2 JD	95	if (hvc->index != 0) {
1466abf2 JD	96	last = &hvc->ops[hvc->index - 1];
ba180fd4	97	if ((last->type == MMAP) &&
c5600490	98	(last->u.mmap.addr + last->u.mmap.len == virt) &&
16dd07bc	99	(last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) &&
ba180fd4	100	(last->u.mmap.offset + last->u.mmap.len == offset)) {
c5600490	101	last->u.mmap.len += len;
07bf731e	102	return 0;
c5600490 JD	103	}
	104	}
	105
1466abf2 JD	106	if (hvc->index == ARRAY_SIZE(hvc->ops)) {
	107	ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
	108	hvc->index = 0;
c5600490 JD	109	}
c5600490 JD	110
1466abf2 JD	111	hvc->ops[hvc->index++] = ((struct host_vm_op)
	112	{ .type = MMAP,
	113	.u = { .mmap = { .addr = virt,
	114	.len = len,
	115	.prot = prot,
	116	.fd = fd,
	117	.offset = offset }
07bf731e BS	118	} });
07bf731e BS	119	return ret;
c5600490 JD	120	}
	121
	122	static int add_munmap(unsigned long addr, unsigned long len,
1466abf2	123	struct host_vm_change *hvc)
c5600490 JD	124	{
c5600490 JD	125	struct host_vm_op *last;
07bf731e	126	int ret = 0;
c5600490	127
284e6d39 RW	128	if ((addr >= STUB_START) && (addr < STUB_END))
	129	return -EINVAL;
	130
1466abf2 JD	131	if (hvc->index != 0) {
1466abf2 JD	132	last = &hvc->ops[hvc->index - 1];
ba180fd4 JD	133	if ((last->type == MUNMAP) &&
ba180fd4 JD	134	(last->u.munmap.addr + last->u.mmap.len == addr)) {
c5600490	135	last->u.munmap.len += len;
07bf731e	136	return 0;
c5600490 JD	137	}
	138	}
	139
1466abf2 JD	140	if (hvc->index == ARRAY_SIZE(hvc->ops)) {
	141	ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
	142	hvc->index = 0;
c5600490 JD	143	}
c5600490 JD	144
1466abf2 JD	145	hvc->ops[hvc->index++] = ((struct host_vm_op)
	146	{ .type = MUNMAP,
	147	.u = { .munmap = { .addr = addr,
	148	.len = len } } });
07bf731e	149	return ret;
c5600490 JD	150	}
c5600490 JD	151
16dd07bc	152	static int add_mprotect(unsigned long addr, unsigned long len,
1466abf2	153	unsigned int prot, struct host_vm_change *hvc)
c5600490 JD	154	{
c5600490 JD	155	struct host_vm_op *last;
07bf731e	156	int ret = 0;
c5600490	157
1466abf2 JD	158	if (hvc->index != 0) {
1466abf2 JD	159	last = &hvc->ops[hvc->index - 1];
ba180fd4	160	if ((last->type == MPROTECT) &&
c5600490	161	(last->u.mprotect.addr + last->u.mprotect.len == addr) &&
ba180fd4	162	(last->u.mprotect.prot == prot)) {
c5600490	163	last->u.mprotect.len += len;
07bf731e	164	return 0;
c5600490 JD	165	}
	166	}
	167
1466abf2 JD	168	if (hvc->index == ARRAY_SIZE(hvc->ops)) {
	169	ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
	170	hvc->index = 0;
c5600490 JD	171	}
c5600490 JD	172
1466abf2 JD	173	hvc->ops[hvc->index++] = ((struct host_vm_op)
	174	{ .type = MPROTECT,
	175	.u = { .mprotect = { .addr = addr,
	176	.len = len,
	177	.prot = prot } } });
07bf731e	178	return ret;
c5600490 JD	179	}
c5600490 JD	180
1da177e4 LT	181	#define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
1da177e4 LT	182
7f0536f8	183	static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
1466abf2 JD	184	unsigned long end,
1466abf2 JD	185	struct host_vm_change *hvc)
7f0536f8 JD	186	{
7f0536f8 JD	187	pte_t *pte;
16dd07bc	188	int r, w, x, prot, ret = 0;
7f0536f8 JD	189
	190	pte = pte_offset_kernel(pmd, addr);
	191	do {
3963333f JD	192	if ((addr >= STUB_START) && (addr < STUB_END))
	193	continue;
	194
7f0536f8 JD	195	r = pte_read(*pte);
	196	w = pte_write(*pte);
	197	x = pte_exec(*pte);
	198	if (!pte_young(*pte)) {
	199	r = 0;
	200	w = 0;
0b4e273f	201	} else if (!pte_dirty(*pte))
7f0536f8	202	w = 0;
0b4e273f	203
16dd07bc JD	204	prot = ((r ? UM_PROT_READ : 0) \| (w ? UM_PROT_WRITE : 0) \|
16dd07bc JD	205	(x ? UM_PROT_EXEC : 0));
1466abf2	206	if (hvc->force \|\| pte_newpage(*pte)) {
ba180fd4	207	if (pte_present(*pte))
7f0536f8	208	ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
1466abf2	209	PAGE_SIZE, prot, hvc);
0b4e273f JD	210	else
	211	ret = add_munmap(addr, PAGE_SIZE, hvc);
	212	} else if (pte_newprot(*pte))
1466abf2	213	ret = add_mprotect(addr, PAGE_SIZE, prot, hvc);
7f0536f8	214	pte = pte_mkuptodate(pte);
909e90d3	215	} while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret));
7f0536f8 JD	216	return ret;
	217	}
	218
	219	static inline int update_pmd_range(pud_t *pud, unsigned long addr,
1466abf2 JD	220	unsigned long end,
1466abf2 JD	221	struct host_vm_change *hvc)
7f0536f8 JD	222	{
	223	pmd_t *pmd;
	224	unsigned long next;
	225	int ret = 0;
	226
	227	pmd = pmd_offset(pud, addr);
	228	do {
	229	next = pmd_addr_end(addr, end);
ba180fd4	230	if (!pmd_present(*pmd)) {
1466abf2 JD	231	if (hvc->force \|\| pmd_newpage(*pmd)) {
1466abf2 JD	232	ret = add_munmap(addr, next - addr, hvc);
7f0536f8 JD	233	pmd_mkuptodate(*pmd);
	234	}
	235	}
1466abf2	236	else ret = update_pte_range(pmd, addr, next, hvc);
909e90d3	237	} while (pmd++, addr = next, ((addr < end) && !ret));
7f0536f8 JD	238	return ret;
	239	}
	240
	241	static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
1466abf2 JD	242	unsigned long end,
1466abf2 JD	243	struct host_vm_change *hvc)
7f0536f8 JD	244	{
	245	pud_t *pud;
	246	unsigned long next;
	247	int ret = 0;
	248
	249	pud = pud_offset(pgd, addr);
	250	do {
	251	next = pud_addr_end(addr, end);
ba180fd4	252	if (!pud_present(*pud)) {
1466abf2 JD	253	if (hvc->force \|\| pud_newpage(*pud)) {
1466abf2 JD	254	ret = add_munmap(addr, next - addr, hvc);
7f0536f8 JD	255	pud_mkuptodate(*pud);
	256	}
	257	}
1466abf2	258	else ret = update_pmd_range(pud, addr, next, hvc);
909e90d3	259	} while (pud++, addr = next, ((addr < end) && !ret));
7f0536f8 JD	260	return ret;
	261	}
	262
1da177e4	263	void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
1466abf2	264	unsigned long end_addr, int force)
1da177e4	265	{
7f0536f8	266	pgd_t *pgd;
1466abf2	267	struct host_vm_change hvc;
7f0536f8	268	unsigned long addr = start_addr, next;
1466abf2	269	int ret = 0;
1da177e4	270
1466abf2	271	hvc = INIT_HVC(mm, force);
7f0536f8 JD	272	pgd = pgd_offset(mm, addr);
	273	do {
	274	next = pgd_addr_end(addr, end_addr);
ba180fd4 JD	275	if (!pgd_present(*pgd)) {
ba180fd4 JD	276	if (force \|\| pgd_newpage(*pgd)) {
1466abf2	277	ret = add_munmap(addr, next - addr, &hvc);
7f0536f8	278	pgd_mkuptodate(*pgd);
5e1f65a6	279	}
5e1f65a6	280	}
1466abf2	281	else ret = update_pud_range(pgd, addr, next, &hvc);
909e90d3	282	} while (pgd++, addr = next, ((addr < end_addr) && !ret));
5e1f65a6	283
ba180fd4	284	if (!ret)
1466abf2	285	ret = do_ops(&hvc, hvc.index, 1);
07bf731e	286
7f0536f8	287	/* This is not an else because ret is modified above */
ba180fd4 JD	288	if (ret) {
ba180fd4 JD	289	printk(KERN_ERR "fix_range_common: failed, killing current "
468f6597 RW	290	"process: %d\n", task_tgid_vnr(current));
	291	/* We are under mmap_sem, release it such that current can terminate */
	292	up_write(&current->mm->mmap_sem);
07bf731e	293	force_sig(SIGKILL, current);
468f6597	294	do_signal();
07bf731e	295	}
1da177e4 LT	296	}
1da177e4 LT	297
c75d053b	298	static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
1da177e4	299	{
5e1f65a6 JD	300	struct mm_struct *mm;
	301	pgd_t *pgd;
	302	pud_t *pud;
	303	pmd_t *pmd;
	304	pte_t *pte;
	305	unsigned long addr, last;
	306	int updated = 0, err;
	307
	308	mm = &init_mm;
ba180fd4	309	for (addr = start; addr < end;) {
5e1f65a6	310	pgd = pgd_offset(mm, addr);
ba180fd4	311	if (!pgd_present(*pgd)) {
5e1f65a6	312	last = ADD_ROUND(addr, PGDIR_SIZE);
ba180fd4	313	if (last > end)
5e1f65a6	314	last = end;
ba180fd4	315	if (pgd_newpage(*pgd)) {
5e1f65a6 JD	316	updated = 1;
	317	err = os_unmap_memory((void *) addr,
	318	last - addr);
ba180fd4	319	if (err < 0)
5e1f65a6 JD	320	panic("munmap failed, errno = %d\n",
	321	-err);
	322	}
	323	addr = last;
	324	continue;
	325	}
	326
	327	pud = pud_offset(pgd, addr);
ba180fd4	328	if (!pud_present(*pud)) {
5e1f65a6	329	last = ADD_ROUND(addr, PUD_SIZE);
ba180fd4	330	if (last > end)
5e1f65a6	331	last = end;
ba180fd4	332	if (pud_newpage(*pud)) {
5e1f65a6 JD	333	updated = 1;
	334	err = os_unmap_memory((void *) addr,
	335	last - addr);
ba180fd4	336	if (err < 0)
5e1f65a6 JD	337	panic("munmap failed, errno = %d\n",
	338	-err);
	339	}
	340	addr = last;
	341	continue;
	342	}
	343
	344	pmd = pmd_offset(pud, addr);
ba180fd4	345	if (!pmd_present(*pmd)) {
5e1f65a6	346	last = ADD_ROUND(addr, PMD_SIZE);
ba180fd4	347	if (last > end)
5e1f65a6	348	last = end;
ba180fd4	349	if (pmd_newpage(*pmd)) {
5e1f65a6 JD	350	updated = 1;
	351	err = os_unmap_memory((void *) addr,
	352	last - addr);
ba180fd4	353	if (err < 0)
5e1f65a6 JD	354	panic("munmap failed, errno = %d\n",
	355	-err);
	356	}
	357	addr = last;
	358	continue;
	359	}
	360
	361	pte = pte_offset_kernel(pmd, addr);
ba180fd4	362	if (!pte_present(pte) \|\| pte_newpage(pte)) {
5e1f65a6 JD	363	updated = 1;
	364	err = os_unmap_memory((void *) addr,
	365	PAGE_SIZE);
ba180fd4	366	if (err < 0)
5e1f65a6 JD	367	panic("munmap failed, errno = %d\n",
5e1f65a6 JD	368	-err);
ba180fd4	369	if (pte_present(*pte))
5e1f65a6 JD	370	map_memory(addr,
	371	pte_val(*pte) & PAGE_MASK,
	372	PAGE_SIZE, 1, 1, 1);
	373	}
ba180fd4	374	else if (pte_newprot(*pte)) {
5e1f65a6 JD	375	updated = 1;
	376	os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
	377	}
	378	addr += PAGE_SIZE;
	379	}
ba180fd4	380	return updated;
1da177e4 LT	381	}
1da177e4 LT	382
77bf4400 JD	383	void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
	384	{
	385	pgd_t *pgd;
	386	pud_t *pud;
	387	pmd_t *pmd;
	388	pte_t *pte;
	389	struct mm_struct *mm = vma->vm_mm;
	390	void *flush = NULL;
	391	int r, w, x, prot, err = 0;
	392	struct mm_id *mm_id;
	393
	394	address &= PAGE_MASK;
	395	pgd = pgd_offset(mm, address);
ba180fd4	396	if (!pgd_present(*pgd))
77bf4400 JD	397	goto kill;
	398
	399	pud = pud_offset(pgd, address);
ba180fd4	400	if (!pud_present(*pud))
77bf4400 JD	401	goto kill;
	402
	403	pmd = pmd_offset(pud, address);
ba180fd4	404	if (!pmd_present(*pmd))
77bf4400 JD	405	goto kill;
	406
	407	pte = pte_offset_kernel(pmd, address);
	408
	409	r = pte_read(*pte);
	410	w = pte_write(*pte);
	411	x = pte_exec(*pte);
	412	if (!pte_young(*pte)) {
	413	r = 0;
	414	w = 0;
	415	} else if (!pte_dirty(*pte)) {
	416	w = 0;
	417	}
	418
6c738ffa	419	mm_id = &mm->context.id;
77bf4400 JD	420	prot = ((r ? UM_PROT_READ : 0) \| (w ? UM_PROT_WRITE : 0) \|
77bf4400 JD	421	(x ? UM_PROT_EXEC : 0));
ba180fd4 JD	422	if (pte_newpage(*pte)) {
ba180fd4 JD	423	if (pte_present(*pte)) {
77bf4400 JD	424	unsigned long long offset;
	425	int fd;
	426
	427	fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset);
	428	err = map(mm_id, address, PAGE_SIZE, prot, fd, offset,
	429	1, &flush);
	430	}
	431	else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush);
	432	}
ba180fd4	433	else if (pte_newprot(*pte))
77bf4400 JD	434	err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush);
77bf4400 JD	435
ba180fd4	436	if (err)
77bf4400 JD	437	goto kill;
	438
	439	pte = pte_mkuptodate(pte);
	440
	441	return;
	442
	443	kill:
ba180fd4	444	printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address);
77bf4400 JD	445	force_sig(SIGKILL, current);
	446	}
	447
1da177e4 LT	448	pgd_t pgd_offset_proc(struct mm_struct mm, unsigned long address)
1da177e4 LT	449	{
ba180fd4	450	return pgd_offset(mm, address);
1da177e4 LT	451	}
	452
	453	pud_t pud_offset_proc(pgd_t pgd, unsigned long address)
	454	{
ba180fd4	455	return pud_offset(pgd, address);
1da177e4 LT	456	}
	457
	458	pmd_t pmd_offset_proc(pud_t pud, unsigned long address)
	459	{
ba180fd4	460	return pmd_offset(pud, address);
1da177e4 LT	461	}
	462
	463	pte_t pte_offset_proc(pmd_t pmd, unsigned long address)
	464	{
ba180fd4	465	return pte_offset_kernel(pmd, address);
1da177e4 LT	466	}
	467
	468	pte_t addr_pte(struct task_struct task, unsigned long addr)
	469	{
5e1f65a6 JD	470	pgd_t *pgd = pgd_offset(task->mm, addr);
	471	pud_t *pud = pud_offset(pgd, addr);
	472	pmd_t *pmd = pmd_offset(pud, addr);
1da177e4	473
ba180fd4	474	return pte_offset_map(pmd, addr);
1da177e4 LT	475	}
1da177e4 LT	476
d67b569f JD	477	void flush_tlb_all(void)
d67b569f JD	478	{
5e1f65a6	479	flush_tlb_mm(current->mm);
d67b569f JD	480	}
	481
	482	void flush_tlb_kernel_range(unsigned long start, unsigned long end)
	483	{
6aa802ce	484	flush_tlb_kernel_range_common(start, end);
d67b569f JD	485	}
	486
	487	void flush_tlb_kernel_vm(void)
	488	{
6aa802ce	489	flush_tlb_kernel_range_common(start_vm, end_vm);
d67b569f JD	490	}
	491
	492	void __flush_tlb_one(unsigned long addr)
	493	{
ba180fd4	494	flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE);
77bf4400 JD	495	}
77bf4400 JD	496
77bf4400 JD	497	static void fix_range(struct mm_struct *mm, unsigned long start_addr,
	498	unsigned long end_addr, int force)
	499	{
1466abf2	500	fix_range_common(mm, start_addr, end_addr, force);
d67b569f JD	501	}
	502
	503	void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	504	unsigned long end)
	505	{
ba180fd4 JD	506	if (vma->vm_mm == NULL)
	507	flush_tlb_kernel_range_common(start, end);
	508	else fix_range(vma->vm_mm, start, end, 0);
d67b569f	509	}
73395a00	510	EXPORT_SYMBOL(flush_tlb_range);
d67b569f	511
0b4e273f JD	512	void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
0b4e273f JD	513	unsigned long end)
d67b569f	514	{
ba180fd4 JD	515	/*
	516	* Don't bother flushing if this address space is about to be
	517	* destroyed.
	518	*/
	519	if (atomic_read(&mm->mm_users) == 0)
	520	return;
77bf4400	521
0b4e273f JD	522	fix_range(mm, start, end, 0);
	523	}
	524
	525	void flush_tlb_mm(struct mm_struct *mm)
	526	{
	527	struct vm_area_struct *vma = mm->mmap;
	528
	529	while (vma != NULL) {
	530	fix_range(mm, vma->vm_start, vma->vm_end, 0);
	531	vma = vma->vm_next;
	532	}
d67b569f JD	533	}
	534
	535	void force_flush_all(void)
	536	{
77bf4400 JD	537	struct mm_struct *mm = current->mm;
	538	struct vm_area_struct *vma = mm->mmap;
	539
ba180fd4	540	while (vma != NULL) {
77bf4400 JD	541	fix_range(mm, vma->vm_start, vma->vm_end, 1);
	542	vma = vma->vm_next;
	543	}
d67b569f	544	}