[deliverable/linux.git] / mm / iov_iter.c

#include <linux/export.h>
#include <linux/uio.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

#define iterate_iovec(i, n, __v, __p, skip, STEP) {	\
	size_t left;					\
	size_t wanted = n;				\
	__p = i->iov;					\
	__v.iov_len = min(n, __p->iov_len - skip);	\
	if (likely(__v.iov_len)) {			\
		__v.iov_base = __p->iov_base + skip;	\
		left = (STEP);				\
		__v.iov_len -= left;			\
		skip += __v.iov_len;			\
		n -= __v.iov_len;			\
	} else {					\
		left = 0;				\
	}						\
	while (unlikely(!left && n)) {			\
		__p++;					\
		__v.iov_len = min(n, __p->iov_len);	\
		if (unlikely(!__v.iov_len))		\
			continue;			\
		__v.iov_base = __p->iov_base;		\
		left = (STEP);				\
		__v.iov_len -= left;			\
		skip = __v.iov_len;			\
		n -= __v.iov_len;			\
	}						\
	n = wanted - n;					\
}

#define iterate_bvec(i, n, __v, __p, skip, STEP) {	\
	size_t wanted = n;				\
	__p = i->bvec;					\
	__v.bv_len = min_t(size_t, n, __p->bv_len - skip);	\
	if (likely(__v.bv_len)) {			\
		__v.bv_page = __p->bv_page;		\
		__v.bv_offset = __p->bv_offset + skip; 	\
		(void)(STEP);				\
		skip += __v.bv_len;			\
		n -= __v.bv_len;			\
	}						\
	while (unlikely(n)) {				\
		__p++;					\
		__v.bv_len = min_t(size_t, n, __p->bv_len);	\
		if (unlikely(!__v.bv_len))		\
			continue;			\
		__v.bv_page = __p->bv_page;		\
		__v.bv_offset = __p->bv_offset;		\
		(void)(STEP);				\
		skip = __v.bv_len;			\
		n -= __v.bv_len;			\
	}						\
	n = wanted;					\
}

#define iterate_all_kinds(i, n, v, I, B) {			\
	size_t skip = i->iov_offset;				\
	if (unlikely(i->type & ITER_BVEC)) {			\
		const struct bio_vec *bvec;			\
		struct bio_vec v;				\
		iterate_bvec(i, n, v, bvec, skip, (B))		\
	} else {						\
		const struct iovec *iov;			\
		struct iovec v;					\
		iterate_iovec(i, n, v, iov, skip, (I))		\
	}							\
}

#define iterate_and_advance(i, n, v, I, B) {			\
	size_t skip = i->iov_offset;				\
	if (unlikely(i->type & ITER_BVEC)) {			\
		const struct bio_vec *bvec;			\
		struct bio_vec v;				\
		iterate_bvec(i, n, v, bvec, skip, (B))		\
		if (skip == bvec->bv_len) {			\
			bvec++;					\
			skip = 0;				\
		}						\
		i->nr_segs -= bvec - i->bvec;			\
		i->bvec = bvec;					\
	} else {						\
		const struct iovec *iov;			\
		struct iovec v;					\
		iterate_iovec(i, n, v, iov, skip, (I))		\
		if (skip == iov->iov_len) {			\
			iov++;					\
			skip = 0;				\
		}						\
		i->nr_segs -= iov - i->iov;			\
		i->iov = iov;					\
	}							\
	i->count -= n;						\
	i->iov_offset = skip;					\
}

static size_t copy_to_iter_iovec(void *from, size_t bytes, struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

	left = __copy_to_user(buf, from, copy);
	copy -= left;
	skip += copy;
	from += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
		left = __copy_to_user(buf, from, copy);
		copy -= left;
		skip = copy;
		from += copy;
		bytes -= copy;
	}

	if (skip == iov->iov_len) {
		iov++;
		skip = 0;
	}
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}

static size_t copy_from_iter_iovec(void *to, size_t bytes, struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

	left = __copy_from_user(to, buf, copy);
	copy -= left;
	skip += copy;
	to += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
		left = __copy_from_user(to, buf, copy);
		copy -= left;
		skip = copy;
		to += copy;
		bytes -= copy;
	}

	if (skip == iov->iov_len) {
		iov++;
		skip = 0;
	}
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}

static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;
	void *kaddr, *from;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

	if (!fault_in_pages_writeable(buf, copy)) {
		kaddr = kmap_atomic(page);
		from = kaddr + offset;

		/* first chunk, usually the only one */
		left = __copy_to_user_inatomic(buf, from, copy);
		copy -= left;
		skip += copy;
		from += copy;
		bytes -= copy;

		while (unlikely(!left && bytes)) {
			iov++;
			buf = iov->iov_base;
			copy = min(bytes, iov->iov_len);
			left = __copy_to_user_inatomic(buf, from, copy);
			copy -= left;
			skip = copy;
			from += copy;
			bytes -= copy;
		}
		if (likely(!bytes)) {
			kunmap_atomic(kaddr);
			goto done;
		}
		offset = from - kaddr;
		buf += copy;
		kunmap_atomic(kaddr);
		copy = min(bytes, iov->iov_len - skip);
	}
	/* Too bad - revert to non-atomic kmap */
	kaddr = kmap(page);
	from = kaddr + offset;
	left = __copy_to_user(buf, from, copy);
	copy -= left;
	skip += copy;
	from += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
		left = __copy_to_user(buf, from, copy);
		copy -= left;
		skip = copy;
		from += copy;
		bytes -= copy;
	}
	kunmap(page);
done:
	if (skip == iov->iov_len) {
		iov++;
		skip = 0;
	}
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}

static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;
	void *kaddr, *to;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

	if (!fault_in_pages_readable(buf, copy)) {
		kaddr = kmap_atomic(page);
		to = kaddr + offset;

		/* first chunk, usually the only one */
		left = __copy_from_user_inatomic(to, buf, copy);
		copy -= left;
		skip += copy;
		to += copy;
		bytes -= copy;

		while (unlikely(!left && bytes)) {
			iov++;
			buf = iov->iov_base;
			copy = min(bytes, iov->iov_len);
			left = __copy_from_user_inatomic(to, buf, copy);
			copy -= left;
			skip = copy;
			to += copy;
			bytes -= copy;
		}
		if (likely(!bytes)) {
			kunmap_atomic(kaddr);
			goto done;
		}
		offset = to - kaddr;
		buf += copy;
		kunmap_atomic(kaddr);
		copy = min(bytes, iov->iov_len - skip);
	}
	/* Too bad - revert to non-atomic kmap */
	kaddr = kmap(page);
	to = kaddr + offset;
	left = __copy_from_user(to, buf, copy);
	copy -= left;
	skip += copy;
	to += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
		left = __copy_from_user(to, buf, copy);
		copy -= left;
		skip = copy;
		to += copy;
		bytes -= copy;
	}
	kunmap(page);
done:
	if (skip == iov->iov_len) {
		iov++;
		skip = 0;
	}
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}

/*
 * Fault in the first iovec of the given iov_iter, to a maximum length
 * of bytes. Returns 0 on success, or non-zero if the memory could not be
 * accessed (ie. because it is an invalid address).
 *
 * writev-intensive code may want this to prefault several iovecs -- that
 * would be possible (callers must not rely on the fact that _only_ the
 * first iovec will be faulted with the current implementation).
 */
int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
{
	if (!(i->type & ITER_BVEC)) {
		char __user *buf = i->iov->iov_base + i->iov_offset;
		bytes = min(bytes, i->iov->iov_len - i->iov_offset);
		return fault_in_pages_readable(buf, bytes);
	}
	return 0;
}
EXPORT_SYMBOL(iov_iter_fault_in_readable);

void iov_iter_init(struct iov_iter *i, int direction,
			const struct iovec *iov, unsigned long nr_segs,
			size_t count)
{
	/* It will get better.  Eventually... */
	if (segment_eq(get_fs(), KERNEL_DS))
		direction |= ITER_KVEC;
	i->type = direction;
	i->iov = iov;
	i->nr_segs = nr_segs;
	i->iov_offset = 0;
	i->count = count;
}
EXPORT_SYMBOL(iov_iter_init);

static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
{
	char *from = kmap_atomic(page);
	memcpy(to, from + offset, len);
	kunmap_atomic(from);
}

static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len)
{
	char *to = kmap_atomic(page);
	memcpy(to + offset, from, len);
	kunmap_atomic(to);
}

static void memzero_page(struct page *page, size_t offset, size_t len)
{
	char *addr = kmap_atomic(page);
	memset(addr + offset, 0, len);
	kunmap_atomic(addr);
}

static size_t copy_to_iter_bvec(void *from, size_t bytes, struct iov_iter *i)
{
	size_t skip, copy, wanted;
	const struct bio_vec *bvec;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	wanted = bytes;
	bvec = i->bvec;
	skip = i->iov_offset;
	copy = min_t(size_t, bytes, bvec->bv_len - skip);

	memcpy_to_page(bvec->bv_page, skip + bvec->bv_offset, from, copy);
	skip += copy;
	from += copy;
	bytes -= copy;
	while (bytes) {
		bvec++;
		copy = min(bytes, (size_t)bvec->bv_len);
		memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, copy);
		skip = copy;
		from += copy;
		bytes -= copy;
	}
	if (skip == bvec->bv_len) {
		bvec++;
		skip = 0;
	}
	i->count -= wanted - bytes;
	i->nr_segs -= bvec - i->bvec;
	i->bvec = bvec;
	i->iov_offset = skip;
	return wanted - bytes;
}

static size_t copy_from_iter_bvec(void *to, size_t bytes, struct iov_iter *i)
{
	size_t skip, copy, wanted;
	const struct bio_vec *bvec;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	wanted = bytes;
	bvec = i->bvec;
	skip = i->iov_offset;

	copy = min(bytes, bvec->bv_len - skip);

	memcpy_from_page(to, bvec->bv_page, bvec->bv_offset + skip, copy);

	to += copy;
	skip += copy;
	bytes -= copy;

	while (bytes) {
		bvec++;
		copy = min(bytes, (size_t)bvec->bv_len);
		memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, copy);
		skip = copy;
		to += copy;
		bytes -= copy;
	}
	if (skip == bvec->bv_len) {
		bvec++;
		skip = 0;
	}
	i->count -= wanted;
	i->nr_segs -= bvec - i->bvec;
	i->bvec = bvec;
	i->iov_offset = skip;
	return wanted;
}

size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i)
{
	if (i->type & ITER_BVEC)
		return copy_to_iter_bvec(addr, bytes, i);
	else
		return copy_to_iter_iovec(addr, bytes, i);
}
EXPORT_SYMBOL(copy_to_iter);

size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
{
	if (i->type & ITER_BVEC)
		return copy_from_iter_bvec(addr, bytes, i);
	else
		return copy_from_iter_iovec(addr, bytes, i);
}
EXPORT_SYMBOL(copy_from_iter);

size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	if (i->type & (ITER_BVEC|ITER_KVEC)) {
		void *kaddr = kmap_atomic(page);
		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
		kunmap_atomic(kaddr);
		return wanted;
	} else
		return copy_page_to_iter_iovec(page, offset, bytes, i);
}
EXPORT_SYMBOL(copy_page_to_iter);

size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	if (i->type & ITER_BVEC) {
		void *kaddr = kmap_atomic(page);
		size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
		kunmap_atomic(kaddr);
		return wanted;
	} else
		return copy_page_from_iter_iovec(page, offset, bytes, i);
}
EXPORT_SYMBOL(copy_page_from_iter);

size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
{
	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	iterate_and_advance(i, bytes, v,
		__clear_user(v.iov_base, v.iov_len),
		memzero_page(v.bv_page, v.bv_offset, v.bv_len)
	)

	return bytes;
}
EXPORT_SYMBOL(iov_iter_zero);

size_t iov_iter_copy_from_user_atomic(struct page *page,
		struct iov_iter *i, unsigned long offset, size_t bytes)
{
	char *kaddr = kmap_atomic(page), *p = kaddr + offset;
	iterate_all_kinds(i, bytes, v,
		__copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
					  v.iov_base, v.iov_len),
		memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
				 v.bv_offset, v.bv_len)
	)
	kunmap_atomic(kaddr);
	return bytes;
}
EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);

void iov_iter_advance(struct iov_iter *i, size_t size)
{
	iterate_and_advance(i, size, v, 0, 0)
}
EXPORT_SYMBOL(iov_iter_advance);

/*
 * Return the count of just the current iov_iter segment.
 */
size_t iov_iter_single_seg_count(const struct iov_iter *i)
{
	if (i->nr_segs == 1)
		return i->count;
	else if (i->type & ITER_BVEC)
		return min(i->count, i->bvec->bv_len - i->iov_offset);
	else
		return min(i->count, i->iov->iov_len - i->iov_offset);
}
EXPORT_SYMBOL(iov_iter_single_seg_count);

unsigned long iov_iter_alignment(const struct iov_iter *i)
{
	unsigned long res = 0;
	size_t size = i->count;

	if (!size)
		return 0;

	iterate_all_kinds(i, size, v,
		(res |= (unsigned long)v.iov_base | v.iov_len, 0),
		res |= v.bv_offset | v.bv_len
	)
	return res;
}
EXPORT_SYMBOL(iov_iter_alignment);

ssize_t iov_iter_get_pages(struct iov_iter *i,
		   struct page **pages, size_t maxsize, unsigned maxpages,
		   size_t *start)
{
	if (maxsize > i->count)
		maxsize = i->count;

	if (!maxsize)
		return 0;

	iterate_all_kinds(i, maxsize, v, ({
		unsigned long addr = (unsigned long)v.iov_base;
		size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
		int n;
		int res;

		if (len > maxpages * PAGE_SIZE)
			len = maxpages * PAGE_SIZE;
		addr &= ~(PAGE_SIZE - 1);
		n = DIV_ROUND_UP(len, PAGE_SIZE);
		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
		if (unlikely(res < 0))
			return res;
		return (res == n ? len : res * PAGE_SIZE) - *start;
	0;}),({
		/* can't be more than PAGE_SIZE */
		*start = v.bv_offset;
		get_page(*pages = v.bv_page);
		return v.bv_len;
	})
	)
	return 0;
}
EXPORT_SYMBOL(iov_iter_get_pages);

static struct page **get_pages_array(size_t n)
{
	struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
	if (!p)
		p = vmalloc(n * sizeof(struct page *));
	return p;
}

ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
		   struct page ***pages, size_t maxsize,
		   size_t *start)
{
	struct page **p;

	if (maxsize > i->count)
		maxsize = i->count;

	if (!maxsize)
		return 0;

	iterate_all_kinds(i, maxsize, v, ({
		unsigned long addr = (unsigned long)v.iov_base;
		size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
		int n;
		int res;

		addr &= ~(PAGE_SIZE - 1);
		n = DIV_ROUND_UP(len, PAGE_SIZE);
		p = get_pages_array(n);
		if (!p)
			return -ENOMEM;
		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
		if (unlikely(res < 0)) {
			kvfree(p);
			return res;
		}
		*pages = p;
		return (res == n ? len : res * PAGE_SIZE) - *start;
	0;}),({
		/* can't be more than PAGE_SIZE */
		*start = v.bv_offset;
		*pages = p = get_pages_array(1);
		if (!p)
			return -ENOMEM;
		get_page(*p = v.bv_page);
		return v.bv_len;
	})
	)
	return 0;
}
EXPORT_SYMBOL(iov_iter_get_pages_alloc);

int iov_iter_npages(const struct iov_iter *i, int maxpages)
{
	size_t size = i->count;
	int npages = 0;

	if (!size)
		return 0;

	iterate_all_kinds(i, size, v, ({
		unsigned long p = (unsigned long)v.iov_base;
		npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
			- p / PAGE_SIZE;
		if (npages >= maxpages)
			return maxpages;
	0;}),({
		npages++;
		if (npages >= maxpages)
			return maxpages;
	})
	)
	return npages;
}
EXPORT_SYMBOL(iov_iter_npages);
Commit	Line	Data
4f18cd31 AV	1	#include <linux/export.h>
	2	#include <linux/uio.h>
	3	#include <linux/pagemap.h>
91f79c43 AV	4	#include <linux/slab.h>
91f79c43 AV	5	#include <linux/vmalloc.h>
4f18cd31	6
04a31165 AV	7	#define iterate_iovec(i, n, __v, __p, skip, STEP) { \
	8	size_t left; \
	9	size_t wanted = n; \
	10	__p = i->iov; \
	11	__v.iov_len = min(n, __p->iov_len - skip); \
	12	if (likely(__v.iov_len)) { \
	13	__v.iov_base = __p->iov_base + skip; \
	14	left = (STEP); \
	15	__v.iov_len -= left; \
	16	skip += __v.iov_len; \
	17	n -= __v.iov_len; \
	18	} else { \
	19	left = 0; \
	20	} \
	21	while (unlikely(!left && n)) { \
	22	__p++; \
	23	__v.iov_len = min(n, __p->iov_len); \
	24	if (unlikely(!__v.iov_len)) \
	25	continue; \
	26	__v.iov_base = __p->iov_base; \
	27	left = (STEP); \
	28	__v.iov_len -= left; \
	29	skip = __v.iov_len; \
	30	n -= __v.iov_len; \
	31	} \
	32	n = wanted - n; \
	33	}
	34
	35	#define iterate_bvec(i, n, __v, __p, skip, STEP) { \
	36	size_t wanted = n; \
	37	__p = i->bvec; \
	38	__v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
	39	if (likely(__v.bv_len)) { \
	40	__v.bv_page = __p->bv_page; \
	41	__v.bv_offset = __p->bv_offset + skip; \
	42	(void)(STEP); \
	43	skip += __v.bv_len; \
	44	n -= __v.bv_len; \
	45	} \
	46	while (unlikely(n)) { \
	47	__p++; \
	48	__v.bv_len = min_t(size_t, n, __p->bv_len); \
	49	if (unlikely(!__v.bv_len)) \
	50	continue; \
	51	__v.bv_page = __p->bv_page; \
	52	__v.bv_offset = __p->bv_offset; \
	53	(void)(STEP); \
	54	skip = __v.bv_len; \
	55	n -= __v.bv_len; \
	56	} \
	57	n = wanted; \
	58	}
	59
	60	#define iterate_all_kinds(i, n, v, I, B) { \
	61	size_t skip = i->iov_offset; \
	62	if (unlikely(i->type & ITER_BVEC)) { \
	63	const struct bio_vec *bvec; \
	64	struct bio_vec v; \
	65	iterate_bvec(i, n, v, bvec, skip, (B)) \
	66	} else { \
	67	const struct iovec *iov; \
	68	struct iovec v; \
	69	iterate_iovec(i, n, v, iov, skip, (I)) \
	70	} \
71	}
72
7ce2a91e AV	73	#define iterate_and_advance(i, n, v, I, B) { \
	74	size_t skip = i->iov_offset; \
	75	if (unlikely(i->type & ITER_BVEC)) { \
	76	const struct bio_vec *bvec; \
	77	struct bio_vec v; \
	78	iterate_bvec(i, n, v, bvec, skip, (B)) \
	79	if (skip == bvec->bv_len) { \
	80	bvec++; \
	81	skip = 0; \
	82	} \
	83	i->nr_segs -= bvec - i->bvec; \
	84	i->bvec = bvec; \
	85	} else { \
	86	const struct iovec *iov; \
	87	struct iovec v; \
	88	iterate_iovec(i, n, v, iov, skip, (I)) \
	89	if (skip == iov->iov_len) { \
	90	iov++; \
	91	skip = 0; \
	92	} \
	93	i->nr_segs -= iov - i->iov; \
	94	i->iov = iov; \
	95	} \
	96	i->count -= n; \
	97	i->iov_offset = skip; \
	98	}
	99
c35e0248 MW	100	static size_t copy_to_iter_iovec(void from, size_t bytes, struct iov_iter i)
	101	{
	102	size_t skip, copy, left, wanted;
	103	const struct iovec *iov;
	104	char __user *buf;
	105
	106	if (unlikely(bytes > i->count))
	107	bytes = i->count;
	108
	109	if (unlikely(!bytes))
	110	return 0;
	111
	112	wanted = bytes;
	113	iov = i->iov;
	114	skip = i->iov_offset;
	115	buf = iov->iov_base + skip;
	116	copy = min(bytes, iov->iov_len - skip);
	117
	118	left = __copy_to_user(buf, from, copy);
	119	copy -= left;
	120	skip += copy;
	121	from += copy;
	122	bytes -= copy;
	123	while (unlikely(!left && bytes)) {
	124	iov++;
	125	buf = iov->iov_base;
	126	copy = min(bytes, iov->iov_len);
	127	left = __copy_to_user(buf, from, copy);
	128	copy -= left;
	129	skip = copy;
	130	from += copy;
	131	bytes -= copy;
	132	}
	133
	134	if (skip == iov->iov_len) {
	135	iov++;
	136	skip = 0;
	137	}
	138	i->count -= wanted - bytes;
	139	i->nr_segs -= iov - i->iov;
	140	i->iov = iov;
	141	i->iov_offset = skip;
	142	return wanted - bytes;
	143	}
	144
	145	static size_t copy_from_iter_iovec(void to, size_t bytes, struct iov_iter i)
	146	{
	147	size_t skip, copy, left, wanted;
	148	const struct iovec *iov;
	149	char __user *buf;
	150
	151	if (unlikely(bytes > i->count))
	152	bytes = i->count;
	153
	154	if (unlikely(!bytes))
	155	return 0;
	156
	157	wanted = bytes;
	158	iov = i->iov;
	159	skip = i->iov_offset;
	160	buf = iov->iov_base + skip;
	161	copy = min(bytes, iov->iov_len - skip);
	162
	163	left = __copy_from_user(to, buf, copy);
164	copy -= left;
165	skip += copy;
166	to += copy;
167	bytes -= copy;
168	while (unlikely(!left && bytes)) {
169	iov++;
170	buf = iov->iov_base;
171	copy = min(bytes, iov->iov_len);
172	left = __copy_from_user(to, buf, copy);
173	copy -= left;
174	skip = copy;
175	to += copy;
176	bytes -= copy;
177	}
178
179	if (skip == iov->iov_len) {
180	iov++;
181	skip = 0;
182	}
183	i->count -= wanted - bytes;
184	i->nr_segs -= iov - i->iov;
185	i->iov = iov;
186	i->iov_offset = skip;
187	return wanted - bytes;
188	}
189
62a8067a	190	static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
4f18cd31 AV	191	struct iov_iter *i)
	192	{
	193	size_t skip, copy, left, wanted;
	194	const struct iovec *iov;
	195	char __user *buf;
	196	void kaddr, from;
	197
	198	if (unlikely(bytes > i->count))
	199	bytes = i->count;
	200
	201	if (unlikely(!bytes))
	202	return 0;
	203
	204	wanted = bytes;
	205	iov = i->iov;
	206	skip = i->iov_offset;
	207	buf = iov->iov_base + skip;
	208	copy = min(bytes, iov->iov_len - skip);
	209
	210	if (!fault_in_pages_writeable(buf, copy)) {
	211	kaddr = kmap_atomic(page);
	212	from = kaddr + offset;
	213
	214	/* first chunk, usually the only one */
	215	left = __copy_to_user_inatomic(buf, from, copy);
	216	copy -= left;
	217	skip += copy;
	218	from += copy;
	219	bytes -= copy;
	220
	221	while (unlikely(!left && bytes)) {
	222	iov++;
	223	buf = iov->iov_base;
	224	copy = min(bytes, iov->iov_len);
	225	left = __copy_to_user_inatomic(buf, from, copy);
	226	copy -= left;
	227	skip = copy;
	228	from += copy;
	229	bytes -= copy;
	230	}
	231	if (likely(!bytes)) {
	232	kunmap_atomic(kaddr);
	233	goto done;
	234	}
	235	offset = from - kaddr;
	236	buf += copy;
	237	kunmap_atomic(kaddr);
	238	copy = min(bytes, iov->iov_len - skip);
	239	}
	240	/* Too bad - revert to non-atomic kmap */
	241	kaddr = kmap(page);
	242	from = kaddr + offset;
	243	left = __copy_to_user(buf, from, copy);
	244	copy -= left;
	245	skip += copy;
	246	from += copy;
	247	bytes -= copy;
	248	while (unlikely(!left && bytes)) {
	249	iov++;
	250	buf = iov->iov_base;
	251	copy = min(bytes, iov->iov_len);
	252	left = __copy_to_user(buf, from, copy);
	253	copy -= left;
	254	skip = copy;
255	from += copy;
256	bytes -= copy;
257	}
258	kunmap(page);
259	done:
81055e58 AV	260	if (skip == iov->iov_len) {
	261	iov++;
	262	skip = 0;
	263	}
4f18cd31 AV	264	i->count -= wanted - bytes;
	265	i->nr_segs -= iov - i->iov;
	266	i->iov = iov;
	267	i->iov_offset = skip;
	268	return wanted - bytes;
	269	}
4f18cd31	270
62a8067a	271	static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
f0d1bec9 AV	272	struct iov_iter *i)
	273	{
	274	size_t skip, copy, left, wanted;
	275	const struct iovec *iov;
	276	char __user *buf;
	277	void kaddr, to;
	278
	279	if (unlikely(bytes > i->count))
	280	bytes = i->count;
	281
	282	if (unlikely(!bytes))
	283	return 0;
	284
	285	wanted = bytes;
	286	iov = i->iov;
	287	skip = i->iov_offset;
	288	buf = iov->iov_base + skip;
	289	copy = min(bytes, iov->iov_len - skip);
	290
	291	if (!fault_in_pages_readable(buf, copy)) {
	292	kaddr = kmap_atomic(page);
	293	to = kaddr + offset;
	294
	295	/* first chunk, usually the only one */
	296	left = __copy_from_user_inatomic(to, buf, copy);
	297	copy -= left;
	298	skip += copy;
	299	to += copy;
	300	bytes -= copy;
	301
	302	while (unlikely(!left && bytes)) {
	303	iov++;
	304	buf = iov->iov_base;
	305	copy = min(bytes, iov->iov_len);
	306	left = __copy_from_user_inatomic(to, buf, copy);
	307	copy -= left;
	308	skip = copy;
	309	to += copy;
	310	bytes -= copy;
	311	}
	312	if (likely(!bytes)) {
	313	kunmap_atomic(kaddr);
	314	goto done;
	315	}
	316	offset = to - kaddr;
	317	buf += copy;
	318	kunmap_atomic(kaddr);
	319	copy = min(bytes, iov->iov_len - skip);
	320	}
	321	/* Too bad - revert to non-atomic kmap */
	322	kaddr = kmap(page);
	323	to = kaddr + offset;
	324	left = __copy_from_user(to, buf, copy);
	325	copy -= left;
	326	skip += copy;
	327	to += copy;
	328	bytes -= copy;
	329	while (unlikely(!left && bytes)) {
	330	iov++;
	331	buf = iov->iov_base;
	332	copy = min(bytes, iov->iov_len);
	333	left = __copy_from_user(to, buf, copy);
	334	copy -= left;
	335	skip = copy;
336	to += copy;
337	bytes -= copy;
338	}
339	kunmap(page);
340	done:
81055e58 AV	341	if (skip == iov->iov_len) {
	342	iov++;
	343	skip = 0;
	344	}
f0d1bec9 AV	345	i->count -= wanted - bytes;
	346	i->nr_segs -= iov - i->iov;
	347	i->iov = iov;
	348	i->iov_offset = skip;
	349	return wanted - bytes;
	350	}
f0d1bec9	351
4f18cd31 AV	352	/*
	353	* Fault in the first iovec of the given iov_iter, to a maximum length
	354	* of bytes. Returns 0 on success, or non-zero if the memory could not be
	355	* accessed (ie. because it is an invalid address).
	356	*
	357	* writev-intensive code may want this to prefault several iovecs -- that
	358	* would be possible (callers must not rely on the fact that _only_ the
	359	* first iovec will be faulted with the current implementation).
	360	*/
	361	int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
	362	{
62a8067a AV	363	if (!(i->type & ITER_BVEC)) {
	364	char __user *buf = i->iov->iov_base + i->iov_offset;
	365	bytes = min(bytes, i->iov->iov_len - i->iov_offset);
	366	return fault_in_pages_readable(buf, bytes);
	367	}
	368	return 0;
4f18cd31 AV	369	}
	370	EXPORT_SYMBOL(iov_iter_fault_in_readable);
	371
71d8e532 AV	372	void iov_iter_init(struct iov_iter *i, int direction,
	373	const struct iovec *iov, unsigned long nr_segs,
	374	size_t count)
	375	{
	376	/* It will get better. Eventually... */
	377	if (segment_eq(get_fs(), KERNEL_DS))
62a8067a	378	direction \|= ITER_KVEC;
71d8e532 AV	379	i->type = direction;
	380	i->iov = iov;
	381	i->nr_segs = nr_segs;
	382	i->iov_offset = 0;
	383	i->count = count;
	384	}
	385	EXPORT_SYMBOL(iov_iter_init);
7b2c99d1	386
62a8067a AV	387	static void memcpy_from_page(char to, struct page page, size_t offset, size_t len)
	388	{
	389	char *from = kmap_atomic(page);
	390	memcpy(to, from + offset, len);
	391	kunmap_atomic(from);
	392	}
	393
	394	static void memcpy_to_page(struct page page, size_t offset, char from, size_t len)
	395	{
	396	char *to = kmap_atomic(page);
	397	memcpy(to + offset, from, len);
	398	kunmap_atomic(to);
	399	}
	400
c35e0248 MW	401	static void memzero_page(struct page *page, size_t offset, size_t len)
	402	{
	403	char *addr = kmap_atomic(page);
	404	memset(addr + offset, 0, len);
	405	kunmap_atomic(addr);
	406	}
	407
	408	static size_t copy_to_iter_bvec(void from, size_t bytes, struct iov_iter i)
62a8067a AV	409	{
	410	size_t skip, copy, wanted;
	411	const struct bio_vec *bvec;
62a8067a AV	412
	413	if (unlikely(bytes > i->count))
	414	bytes = i->count;
	415
	416	if (unlikely(!bytes))
	417	return 0;
	418
	419	wanted = bytes;
	420	bvec = i->bvec;
	421	skip = i->iov_offset;
	422	copy = min_t(size_t, bytes, bvec->bv_len - skip);
	423
62a8067a AV	424	memcpy_to_page(bvec->bv_page, skip + bvec->bv_offset, from, copy);
	425	skip += copy;
	426	from += copy;
	427	bytes -= copy;
	428	while (bytes) {
	429	bvec++;
	430	copy = min(bytes, (size_t)bvec->bv_len);
	431	memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, copy);
	432	skip = copy;
	433	from += copy;
	434	bytes -= copy;
	435	}
62a8067a AV	436	if (skip == bvec->bv_len) {
	437	bvec++;
	438	skip = 0;
	439	}
	440	i->count -= wanted - bytes;
	441	i->nr_segs -= bvec - i->bvec;
	442	i->bvec = bvec;
	443	i->iov_offset = skip;
	444	return wanted - bytes;
	445	}
	446
c35e0248	447	static size_t copy_from_iter_bvec(void to, size_t bytes, struct iov_iter i)
62a8067a AV	448	{
	449	size_t skip, copy, wanted;
	450	const struct bio_vec *bvec;
62a8067a AV	451
	452	if (unlikely(bytes > i->count))
	453	bytes = i->count;
	454
	455	if (unlikely(!bytes))
	456	return 0;
	457
	458	wanted = bytes;
	459	bvec = i->bvec;
	460	skip = i->iov_offset;
	461
62a8067a AV	462	copy = min(bytes, bvec->bv_len - skip);
	463
	464	memcpy_from_page(to, bvec->bv_page, bvec->bv_offset + skip, copy);
	465
	466	to += copy;
	467	skip += copy;
	468	bytes -= copy;
	469
	470	while (bytes) {
	471	bvec++;
	472	copy = min(bytes, (size_t)bvec->bv_len);
	473	memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, copy);
	474	skip = copy;
	475	to += copy;
	476	bytes -= copy;
	477	}
62a8067a AV	478	if (skip == bvec->bv_len) {
	479	bvec++;
	480	skip = 0;
	481	}
	482	i->count -= wanted;
	483	i->nr_segs -= bvec - i->bvec;
	484	i->bvec = bvec;
	485	i->iov_offset = skip;
	486	return wanted;
	487	}
	488
d271524a	489	size_t copy_to_iter(void addr, size_t bytes, struct iov_iter i)
c35e0248	490	{
d271524a AV	491	if (i->type & ITER_BVEC)
	492	return copy_to_iter_bvec(addr, bytes, i);
	493	else
	494	return copy_to_iter_iovec(addr, bytes, i);
c35e0248	495	}
d271524a	496	EXPORT_SYMBOL(copy_to_iter);
c35e0248	497
d271524a	498	size_t copy_from_iter(void addr, size_t bytes, struct iov_iter i)
c35e0248	499	{
d271524a AV	500	if (i->type & ITER_BVEC)
	501	return copy_from_iter_bvec(addr, bytes, i);
	502	else
	503	return copy_from_iter_iovec(addr, bytes, i);
c35e0248	504	}
d271524a	505	EXPORT_SYMBOL(copy_from_iter);
c35e0248	506
62a8067a AV	507	size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
	508	struct iov_iter *i)
	509	{
d271524a AV	510	if (i->type & (ITER_BVEC\|ITER_KVEC)) {
	511	void *kaddr = kmap_atomic(page);
	512	size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
	513	kunmap_atomic(kaddr);
	514	return wanted;
	515	} else
62a8067a AV	516	return copy_page_to_iter_iovec(page, offset, bytes, i);
	517	}
	518	EXPORT_SYMBOL(copy_page_to_iter);
	519
	520	size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
	521	struct iov_iter *i)
	522	{
d271524a AV	523	if (i->type & ITER_BVEC) {
	524	void *kaddr = kmap_atomic(page);
	525	size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
	526	kunmap_atomic(kaddr);
	527	return wanted;
	528	} else
62a8067a AV	529	return copy_page_from_iter_iovec(page, offset, bytes, i);
	530	}
	531	EXPORT_SYMBOL(copy_page_from_iter);
	532
c35e0248 MW	533	size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
c35e0248 MW	534	{
8442fa46 AV	535	if (unlikely(bytes > i->count))
	536	bytes = i->count;
	537
	538	if (unlikely(!bytes))
	539	return 0;
	540
	541	iterate_and_advance(i, bytes, v,
	542	__clear_user(v.iov_base, v.iov_len),
	543	memzero_page(v.bv_page, v.bv_offset, v.bv_len)
	544	)
	545
	546	return bytes;
c35e0248 MW	547	}
	548	EXPORT_SYMBOL(iov_iter_zero);
	549
62a8067a AV	550	size_t iov_iter_copy_from_user_atomic(struct page *page,
	551	struct iov_iter *i, unsigned long offset, size_t bytes)
	552	{
04a31165 AV	553	char kaddr = kmap_atomic(page), p = kaddr + offset;
	554	iterate_all_kinds(i, bytes, v,
	555	__copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
	556	v.iov_base, v.iov_len),
	557	memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
	558	v.bv_offset, v.bv_len)
	559	)
	560	kunmap_atomic(kaddr);
	561	return bytes;
62a8067a AV	562	}
	563	EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
	564
	565	void iov_iter_advance(struct iov_iter *i, size_t size)
	566	{
7ce2a91e	567	iterate_and_advance(i, size, v, 0, 0)
62a8067a AV	568	}
	569	EXPORT_SYMBOL(iov_iter_advance);
	570
	571	/*
	572	* Return the count of just the current iov_iter segment.
	573	*/
	574	size_t iov_iter_single_seg_count(const struct iov_iter *i)
	575	{
	576	if (i->nr_segs == 1)
	577	return i->count;
	578	else if (i->type & ITER_BVEC)
62a8067a	579	return min(i->count, i->bvec->bv_len - i->iov_offset);
ad0eab92 PM	580	else
ad0eab92 PM	581	return min(i->count, i->iov->iov_len - i->iov_offset);
62a8067a AV	582	}
	583	EXPORT_SYMBOL(iov_iter_single_seg_count);
	584
	585	unsigned long iov_iter_alignment(const struct iov_iter *i)
	586	{
04a31165 AV	587	unsigned long res = 0;
	588	size_t size = i->count;
	589
	590	if (!size)
	591	return 0;
	592
	593	iterate_all_kinds(i, size, v,
	594	(res \|= (unsigned long)v.iov_base \| v.iov_len, 0),
	595	res \|= v.bv_offset \| v.bv_len
	596	)
	597	return res;
62a8067a AV	598	}
	599	EXPORT_SYMBOL(iov_iter_alignment);
	600
	601	ssize_t iov_iter_get_pages(struct iov_iter *i,
2c80929c	602	struct page **pages, size_t maxsize, unsigned maxpages,
62a8067a AV	603	size_t *start)
62a8067a AV	604	{
e5393fae AV	605	if (maxsize > i->count)
	606	maxsize = i->count;
	607
	608	if (!maxsize)
	609	return 0;
	610
	611	iterate_all_kinds(i, maxsize, v, ({
	612	unsigned long addr = (unsigned long)v.iov_base;
	613	size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
	614	int n;
	615	int res;
	616
	617	if (len > maxpages * PAGE_SIZE)
	618	len = maxpages * PAGE_SIZE;
	619	addr &= ~(PAGE_SIZE - 1);
	620	n = DIV_ROUND_UP(len, PAGE_SIZE);
	621	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
	622	if (unlikely(res < 0))
	623	return res;
	624	return (res == n ? len : res * PAGE_SIZE) - *start;
	625	0;}),({
	626	/* can't be more than PAGE_SIZE */
	627	*start = v.bv_offset;
	628	get_page(*pages = v.bv_page);
	629	return v.bv_len;
	630	})
	631	)
	632	return 0;
62a8067a AV	633	}
	634	EXPORT_SYMBOL(iov_iter_get_pages);
	635
1b17f1f2 AV	636	static struct page **get_pages_array(size_t n)
	637	{
	638	struct page *p = kmalloc(n sizeof(struct page *), GFP_KERNEL);
	639	if (!p)
	640	p = vmalloc(n * sizeof(struct page *));
	641	return p;
	642	}
	643
62a8067a AV	644	ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
	645	struct page ***pages, size_t maxsize,
	646	size_t *start)
	647	{
1b17f1f2 AV	648	struct page **p;
	649
	650	if (maxsize > i->count)
	651	maxsize = i->count;
	652
	653	if (!maxsize)
	654	return 0;
	655
	656	iterate_all_kinds(i, maxsize, v, ({
	657	unsigned long addr = (unsigned long)v.iov_base;
	658	size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
	659	int n;
	660	int res;
	661
	662	addr &= ~(PAGE_SIZE - 1);
	663	n = DIV_ROUND_UP(len, PAGE_SIZE);
	664	p = get_pages_array(n);
	665	if (!p)
	666	return -ENOMEM;
	667	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
	668	if (unlikely(res < 0)) {
	669	kvfree(p);
	670	return res;
	671	}
	672	*pages = p;
	673	return (res == n ? len : res * PAGE_SIZE) - *start;
	674	0;}),({
	675	/* can't be more than PAGE_SIZE */
	676	*start = v.bv_offset;
	677	*pages = p = get_pages_array(1);
	678	if (!p)
	679	return -ENOMEM;
	680	get_page(*p = v.bv_page);
	681	return v.bv_len;
	682	})
	683	)
	684	return 0;
62a8067a AV	685	}
	686	EXPORT_SYMBOL(iov_iter_get_pages_alloc);
	687
	688	int iov_iter_npages(const struct iov_iter *i, int maxpages)
	689	{
e0f2dc40 AV	690	size_t size = i->count;
	691	int npages = 0;
	692
	693	if (!size)
	694	return 0;
	695
	696	iterate_all_kinds(i, size, v, ({
	697	unsigned long p = (unsigned long)v.iov_base;
	698	npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
	699	- p / PAGE_SIZE;
	700	if (npages >= maxpages)
	701	return maxpages;
	702	0;}),({
	703	npages++;
	704	if (npages >= maxpages)
	705	return maxpages;
	706	})
	707	)
	708	return npages;
62a8067a	709	}
f67da30c	710	EXPORT_SYMBOL(iov_iter_npages);