[deliverable/linux.git] / include / linux / radix-tree.h

/*
 * Copyright (C) 2001 Momchil Velikov
 * Portions Copyright (C) 2001 Christoph Hellwig
 * Copyright (C) 2006 Nick Piggin
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2, or (at
 * your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
#ifndef _LINUX_RADIX_TREE_H
#define _LINUX_RADIX_TREE_H

#include <linux/preempt.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/rcupdate.h>

/*
 * An indirect pointer (root->rnode pointing to a radix_tree_node, rather
 * than a data item) is signalled by the low bit set in the root->rnode
 * pointer.
 *
 * In this case root->height is > 0, but the indirect pointer tests are
 * needed for RCU lookups (because root->height is unreliable). The only
 * time callers need worry about this is when doing a lookup_slot under
 * RCU.
 *
 * Indirect pointer in fact is also used to tag the last pointer of a node
 * when it is shrunk, before we rcu free the node. See shrink code for
 * details.
 */
#define RADIX_TREE_INDIRECT_PTR		1
/*
 * A common use of the radix tree is to store pointers to struct pages;
 * but shmem/tmpfs needs also to store swap entries in the same tree:
 * those are marked as exceptional entries to distinguish them.
 * EXCEPTIONAL_ENTRY tests the bit, EXCEPTIONAL_SHIFT shifts content past it.
 */
#define RADIX_TREE_EXCEPTIONAL_ENTRY	2
#define RADIX_TREE_EXCEPTIONAL_SHIFT	2

#define radix_tree_indirect_to_ptr(ptr) \
	radix_tree_indirect_to_ptr((void __force *)(ptr))

static inline int radix_tree_is_indirect_ptr(void *ptr)
{
	return (int)((unsigned long)ptr & RADIX_TREE_INDIRECT_PTR);
}

/*** radix-tree API starts here ***/

#define RADIX_TREE_MAX_TAGS 3

/* root tags are stored in gfp_mask, shifted by __GFP_BITS_SHIFT */
struct radix_tree_root {
	unsigned int		height;
	gfp_t			gfp_mask;
	struct radix_tree_node	__rcu *rnode;
};

#define RADIX_TREE_INIT(mask)	{					\
	.height = 0,							\
	.gfp_mask = (mask),						\
	.rnode = NULL,							\
}

#define RADIX_TREE(name, mask) \
	struct radix_tree_root name = RADIX_TREE_INIT(mask)

#define INIT_RADIX_TREE(root, mask)					\
do {									\
	(root)->height = 0;						\
	(root)->gfp_mask = (mask);					\
	(root)->rnode = NULL;						\
} while (0)

/**
 * Radix-tree synchronization
 *
 * The radix-tree API requires that users provide all synchronisation (with
 * specific exceptions, noted below).
 *
 * Synchronization of access to the data items being stored in the tree, and
 * management of their lifetimes must be completely managed by API users.
 *
 * For API usage, in general,
 * - any function _modifying_ the tree or tags (inserting or deleting
 *   items, setting or clearing tags) must exclude other modifications, and
 *   exclude any functions reading the tree.
 * - any function _reading_ the tree or tags (looking up items or tags,
 *   gang lookups) must exclude modifications to the tree, but may occur
 *   concurrently with other readers.
 *
 * The notable exceptions to this rule are the following functions:
 * radix_tree_lookup
 * radix_tree_lookup_slot
 * radix_tree_tag_get
 * radix_tree_gang_lookup
 * radix_tree_gang_lookup_slot
 * radix_tree_gang_lookup_tag
 * radix_tree_gang_lookup_tag_slot
 * radix_tree_tagged
 *
 * The first 7 functions are able to be called locklessly, using RCU. The
 * caller must ensure calls to these functions are made within rcu_read_lock()
 * regions. Other readers (lock-free or otherwise) and modifications may be
 * running concurrently.
 *
 * It is still required that the caller manage the synchronization and lifetimes
 * of the items. So if RCU lock-free lookups are used, typically this would mean
 * that the items have their own locks, or are amenable to lock-free access; and
 * that the items are freed by RCU (or only freed after having been deleted from
 * the radix tree *and* a synchronize_rcu() grace period).
 *
 * (Note, rcu_assign_pointer and rcu_dereference are not needed to control
 * access to data items when inserting into or looking up from the radix tree)
 *
 * Note that the value returned by radix_tree_tag_get() may not be relied upon
 * if only the RCU read lock is held.  Functions to set/clear tags and to
 * delete nodes running concurrently with it may affect its result such that
 * two consecutive reads in the same locked section may return different
 * values.  If reliability is required, modification functions must also be
 * excluded from concurrency.
 *
 * radix_tree_tagged is able to be called without locking or RCU.
 */

/**
 * radix_tree_deref_slot	- dereference a slot
 * @pslot:	pointer to slot, returned by radix_tree_lookup_slot
 * Returns:	item that was stored in that slot with any direct pointer flag
 *		removed.
 *
 * For use with radix_tree_lookup_slot().  Caller must hold tree at least read
 * locked across slot lookup and dereference. Not required if write lock is
 * held (ie. items cannot be concurrently inserted).
 *
 * radix_tree_deref_retry must be used to confirm validity of the pointer if
 * only the read lock is held.
 */
static inline void *radix_tree_deref_slot(void **pslot)
{
	return rcu_dereference(*pslot);
}

/**
 * radix_tree_deref_slot_protected	- dereference a slot without RCU lock but with tree lock held
 * @pslot:	pointer to slot, returned by radix_tree_lookup_slot
 * Returns:	item that was stored in that slot with any direct pointer flag
 *		removed.
 *
 * Similar to radix_tree_deref_slot but only used during migration when a pages
 * mapping is being moved. The caller does not hold the RCU read lock but it
 * must hold the tree lock to prevent parallel updates.
 */
static inline void *radix_tree_deref_slot_protected(void **pslot,
							spinlock_t *treelock)
{
	return rcu_dereference_protected(*pslot, lockdep_is_held(treelock));
}

/**
 * radix_tree_deref_retry	- check radix_tree_deref_slot
 * @arg:	pointer returned by radix_tree_deref_slot
 * Returns:	0 if retry is not required, otherwise retry is required
 *
 * radix_tree_deref_retry must be used with radix_tree_deref_slot.
 */
static inline int radix_tree_deref_retry(void *arg)
{
	return unlikely((unsigned long)arg & RADIX_TREE_INDIRECT_PTR);
}

/**
 * radix_tree_exceptional_entry	- radix_tree_deref_slot gave exceptional entry?
 * @arg:	value returned by radix_tree_deref_slot
 * Returns:	0 if well-aligned pointer, non-0 if exceptional entry.
 */
static inline int radix_tree_exceptional_entry(void *arg)
{
	/* Not unlikely because radix_tree_exception often tested first */
	return (unsigned long)arg & RADIX_TREE_EXCEPTIONAL_ENTRY;
}

/**
 * radix_tree_exception	- radix_tree_deref_slot returned either exception?
 * @arg:	value returned by radix_tree_deref_slot
 * Returns:	0 if well-aligned pointer, non-0 if either kind of exception.
 */
static inline int radix_tree_exception(void *arg)
{
	return unlikely((unsigned long)arg &
		(RADIX_TREE_INDIRECT_PTR | RADIX_TREE_EXCEPTIONAL_ENTRY));
}

/**
 * radix_tree_replace_slot	- replace item in a slot
 * @pslot:	pointer to slot, returned by radix_tree_lookup_slot
 * @item:	new item to store in the slot.
 *
 * For use with radix_tree_lookup_slot().  Caller must hold tree write locked
 * across slot lookup and replacement.
 */
static inline void radix_tree_replace_slot(void **pslot, void *item)
{
	BUG_ON(radix_tree_is_indirect_ptr(item));
	rcu_assign_pointer(*pslot, item);
}

int radix_tree_insert(struct radix_tree_root *, unsigned long, void *);
void *radix_tree_lookup(struct radix_tree_root *, unsigned long);
void **radix_tree_lookup_slot(struct radix_tree_root *, unsigned long);
void *radix_tree_delete(struct radix_tree_root *, unsigned long);
unsigned int
radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
			unsigned long first_index, unsigned int max_items);
unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
			void ***results, unsigned long *indices,
			unsigned long first_index, unsigned int max_items);
unsigned long radix_tree_next_hole(struct radix_tree_root *root,
				unsigned long index, unsigned long max_scan);
unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
				unsigned long index, unsigned long max_scan);
int radix_tree_preload(gfp_t gfp_mask);
void radix_tree_init(void);
void *radix_tree_tag_set(struct radix_tree_root *root,
			unsigned long index, unsigned int tag);
void *radix_tree_tag_clear(struct radix_tree_root *root,
			unsigned long index, unsigned int tag);
int radix_tree_tag_get(struct radix_tree_root *root,
			unsigned long index, unsigned int tag);
unsigned int
radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
		unsigned long first_index, unsigned int max_items,
		unsigned int tag);
unsigned int
radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,
		unsigned long first_index, unsigned int max_items,
		unsigned int tag);
unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
		unsigned long *first_indexp, unsigned long last_index,
		unsigned long nr_to_tag,
		unsigned int fromtag, unsigned int totag);
int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag);
unsigned long radix_tree_locate_item(struct radix_tree_root *root, void *item);

static inline void radix_tree_preload_end(void)
{
	preempt_enable();
}

#endif /* _LINUX_RADIX_TREE_H */
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Copyright (C) 2001 Momchil Velikov
	3	* Portions Copyright (C) 2001 Christoph Hellwig
7cf9c2c7	4	* Copyright (C) 2006 Nick Piggin
1da177e4 LT	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License as
	8	* published by the Free Software Foundation; either version 2, or (at
	9	* your option) any later version.
	10	*
	11	* This program is distributed in the hope that it will be useful, but
	12	* WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* along with this program; if not, write to the Free Software
	18	* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	19	*/
	20	#ifndef _LINUX_RADIX_TREE_H
	21	#define _LINUX_RADIX_TREE_H
	22
	23	#include <linux/preempt.h>
	24	#include <linux/types.h>
7cf9c2c7 NP	25	#include <linux/kernel.h>
	26	#include <linux/rcupdate.h>
	27
	28	/*
c0bc9875 NP	29	* An indirect pointer (root->rnode pointing to a radix_tree_node, rather
	30	* than a data item) is signalled by the low bit set in the root->rnode
	31	* pointer.
7cf9c2c7	32	*
c0bc9875 NP	33	* In this case root->height is > 0, but the indirect pointer tests are
	34	* needed for RCU lookups (because root->height is unreliable). The only
	35	* time callers need worry about this is when doing a lookup_slot under
	36	* RCU.
27d20fdd NP	37	*
	38	* Indirect pointer in fact is also used to tag the last pointer of a node
	39	* when it is shrunk, before we rcu free the node. See shrink code for
	40	* details.
7cf9c2c7	41	*/
6328650b HD	42	#define RADIX_TREE_INDIRECT_PTR 1
	43	/*
	44	* A common use of the radix tree is to store pointers to struct pages;
	45	* but shmem/tmpfs needs also to store swap entries in the same tree:
	46	* those are marked as exceptional entries to distinguish them.
	47	* EXCEPTIONAL_ENTRY tests the bit, EXCEPTIONAL_SHIFT shifts content past it.
	48	*/
	49	#define RADIX_TREE_EXCEPTIONAL_ENTRY 2
	50	#define RADIX_TREE_EXCEPTIONAL_SHIFT 2
7cf9c2c7	51
a1115570 AB	52	#define radix_tree_indirect_to_ptr(ptr) \
a1115570 AB	53	radix_tree_indirect_to_ptr((void __force *)(ptr))
7cf9c2c7	54
c0bc9875	55	static inline int radix_tree_is_indirect_ptr(void *ptr)
7cf9c2c7	56	{
c0bc9875	57	return (int)((unsigned long)ptr & RADIX_TREE_INDIRECT_PTR);
7cf9c2c7 NP	58	}
	59
	60	/* radix-tree API starts here */
1da177e4	61
f446daae	62	#define RADIX_TREE_MAX_TAGS 3
612d6c19 NP	63
612d6c19 NP	64	/* root tags are stored in gfp_mask, shifted by __GFP_BITS_SHIFT */
1da177e4 LT	65	struct radix_tree_root {
1da177e4 LT	66	unsigned int height;
fd4f2df2	67	gfp_t gfp_mask;
a1115570	68	struct radix_tree_node __rcu *rnode;
1da177e4 LT	69	};
	70
	71	#define RADIX_TREE_INIT(mask) { \
	72	.height = 0, \
	73	.gfp_mask = (mask), \
	74	.rnode = NULL, \
	75	}
	76
	77	#define RADIX_TREE(name, mask) \
	78	struct radix_tree_root name = RADIX_TREE_INIT(mask)
	79
	80	#define INIT_RADIX_TREE(root, mask) \
	81	do { \
	82	(root)->height = 0; \
	83	(root)->gfp_mask = (mask); \
	84	(root)->rnode = NULL; \
	85	} while (0)
	86
7cf9c2c7 NP	87	/**
	88	* Radix-tree synchronization
	89	*
	90	* The radix-tree API requires that users provide all synchronisation (with
	91	* specific exceptions, noted below).
	92	*
	93	* Synchronization of access to the data items being stored in the tree, and
	94	* management of their lifetimes must be completely managed by API users.
	95	*
	96	* For API usage, in general,
59c51591	97	* - any function _modifying_ the tree or tags (inserting or deleting
eb8dc5e7	98	* items, setting or clearing tags) must exclude other modifications, and
7cf9c2c7	99	* exclude any functions reading the tree.
59c51591	100	* - any function _reading_ the tree or tags (looking up items or tags,
7cf9c2c7 NP	101	* gang lookups) must exclude modifications to the tree, but may occur
	102	* concurrently with other readers.
	103	*
	104	* The notable exceptions to this rule are the following functions:
	105	* radix_tree_lookup
47feff2c	106	* radix_tree_lookup_slot
7cf9c2c7 NP	107	* radix_tree_tag_get
7cf9c2c7 NP	108	* radix_tree_gang_lookup
47feff2c	109	* radix_tree_gang_lookup_slot
7cf9c2c7	110	* radix_tree_gang_lookup_tag
47feff2c	111	* radix_tree_gang_lookup_tag_slot
7cf9c2c7 NP	112	* radix_tree_tagged
7cf9c2c7 NP	113	*
47feff2c	114	* The first 7 functions are able to be called locklessly, using RCU. The
7cf9c2c7 NP	115	* caller must ensure calls to these functions are made within rcu_read_lock()
	116	* regions. Other readers (lock-free or otherwise) and modifications may be
	117	* running concurrently.
	118	*
	119	* It is still required that the caller manage the synchronization and lifetimes
	120	* of the items. So if RCU lock-free lookups are used, typically this would mean
	121	* that the items have their own locks, or are amenable to lock-free access; and
	122	* that the items are freed by RCU (or only freed after having been deleted from
	123	* the radix tree and a synchronize_rcu() grace period).
	124	*
	125	* (Note, rcu_assign_pointer and rcu_dereference are not needed to control
	126	* access to data items when inserting into or looking up from the radix tree)
	127	*
ce82653d DH	128	* Note that the value returned by radix_tree_tag_get() may not be relied upon
	129	* if only the RCU read lock is held. Functions to set/clear tags and to
	130	* delete nodes running concurrently with it may affect its result such that
	131	* two consecutive reads in the same locked section may return different
	132	* values. If reliability is required, modification functions must also be
	133	* excluded from concurrency.
	134	*
7cf9c2c7 NP	135	* radix_tree_tagged is able to be called without locking or RCU.
	136	*/
	137
	138	/**
	139	* radix_tree_deref_slot - dereference a slot
	140	* @pslot: pointer to slot, returned by radix_tree_lookup_slot
	141	* Returns: item that was stored in that slot with any direct pointer flag
	142	* removed.
	143	*
	144	* For use with radix_tree_lookup_slot(). Caller must hold tree at least read
27d20fdd NP	145	* locked across slot lookup and dereference. Not required if write lock is
	146	* held (ie. items cannot be concurrently inserted).
	147	*
	148	* radix_tree_deref_retry must be used to confirm validity of the pointer if
	149	* only the read lock is held.
7cf9c2c7 NP	150	*/
	151	static inline void radix_tree_deref_slot(void *pslot)
	152	{
27d20fdd	153	return rcu_dereference(*pslot);
7cf9c2c7	154	}
27d20fdd	155
29c1f677 MG	156	/**
	157	* radix_tree_deref_slot_protected - dereference a slot without RCU lock but with tree lock held
	158	* @pslot: pointer to slot, returned by radix_tree_lookup_slot
	159	* Returns: item that was stored in that slot with any direct pointer flag
	160	* removed.
	161	*
	162	* Similar to radix_tree_deref_slot but only used during migration when a pages
	163	* mapping is being moved. The caller does not hold the RCU read lock but it
	164	* must hold the tree lock to prevent parallel updates.
	165	*/
	166	static inline void radix_tree_deref_slot_protected(void *pslot,
	167	spinlock_t *treelock)
	168	{
	169	return rcu_dereference_protected(*pslot, lockdep_is_held(treelock));
	170	}
	171
27d20fdd NP	172	/**
	173	* radix_tree_deref_retry - check radix_tree_deref_slot
	174	* @arg: pointer returned by radix_tree_deref_slot
	175	* Returns: 0 if retry is not required, otherwise retry is required
	176	*
	177	* radix_tree_deref_retry must be used with radix_tree_deref_slot.
	178	*/
	179	static inline int radix_tree_deref_retry(void *arg)
	180	{
	181	return unlikely((unsigned long)arg & RADIX_TREE_INDIRECT_PTR);
	182	}
	183
6328650b HD	184	/**
	185	* radix_tree_exceptional_entry - radix_tree_deref_slot gave exceptional entry?
	186	* @arg: value returned by radix_tree_deref_slot
	187	* Returns: 0 if well-aligned pointer, non-0 if exceptional entry.
	188	*/
	189	static inline int radix_tree_exceptional_entry(void *arg)
	190	{
	191	/* Not unlikely because radix_tree_exception often tested first */
	192	return (unsigned long)arg & RADIX_TREE_EXCEPTIONAL_ENTRY;
	193	}
	194
	195	/**
	196	* radix_tree_exception - radix_tree_deref_slot returned either exception?
	197	* @arg: value returned by radix_tree_deref_slot
	198	* Returns: 0 if well-aligned pointer, non-0 if either kind of exception.
	199	*/
	200	static inline int radix_tree_exception(void *arg)
	201	{
	202	return unlikely((unsigned long)arg &
	203	(RADIX_TREE_INDIRECT_PTR \| RADIX_TREE_EXCEPTIONAL_ENTRY));
	204	}
	205
7cf9c2c7 NP	206	/**
	207	* radix_tree_replace_slot - replace item in a slot
	208	* @pslot: pointer to slot, returned by radix_tree_lookup_slot
	209	* @item: new item to store in the slot.
	210	*
	211	* For use with radix_tree_lookup_slot(). Caller must hold tree write locked
	212	* across slot lookup and replacement.
	213	*/
	214	static inline void radix_tree_replace_slot(void *pslot, void item)
	215	{
c0bc9875 NP	216	BUG_ON(radix_tree_is_indirect_ptr(item));
c0bc9875 NP	217	rcu_assign_pointer(*pslot, item);
7cf9c2c7 NP	218	}
7cf9c2c7 NP	219
1da177e4 LT	220	int radix_tree_insert(struct radix_tree_root , unsigned long, void );
1da177e4 LT	221	void radix_tree_lookup(struct radix_tree_root , unsigned long);
a4331366	222	void *radix_tree_lookup_slot(struct radix_tree_root , unsigned long);
1da177e4 LT	223	void radix_tree_delete(struct radix_tree_root , unsigned long);
	224	unsigned int
	225	radix_tree_gang_lookup(struct radix_tree_root root, void *results,
	226	unsigned long first_index, unsigned int max_items);
6328650b HD	227	unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
6328650b HD	228	void **results, unsigned long indices,
47feff2c	229	unsigned long first_index, unsigned int max_items);
6df8ba4f FW	230	unsigned long radix_tree_next_hole(struct radix_tree_root *root,
6df8ba4f FW	231	unsigned long index, unsigned long max_scan);
dc566127 WF	232	unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
dc566127 WF	233	unsigned long index, unsigned long max_scan);
dd0fc66f	234	int radix_tree_preload(gfp_t gfp_mask);
1da177e4 LT	235	void radix_tree_init(void);
1da177e4 LT	236	void radix_tree_tag_set(struct radix_tree_root root,
daff89f3	237	unsigned long index, unsigned int tag);
1da177e4	238	void radix_tree_tag_clear(struct radix_tree_root root,
daff89f3	239	unsigned long index, unsigned int tag);
1da177e4	240	int radix_tree_tag_get(struct radix_tree_root *root,
daff89f3	241	unsigned long index, unsigned int tag);
1da177e4 LT	242	unsigned int
1da177e4 LT	243	radix_tree_gang_lookup_tag(struct radix_tree_root root, void *results,
daff89f3 JC	244	unsigned long first_index, unsigned int max_items,
daff89f3 JC	245	unsigned int tag);
47feff2c NP	246	unsigned int
	247	radix_tree_gang_lookup_tag_slot(struct radix_tree_root root, void **results,
	248	unsigned long first_index, unsigned int max_items,
	249	unsigned int tag);
ebf8aa44 JK	250	unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
	251	unsigned long *first_indexp, unsigned long last_index,
	252	unsigned long nr_to_tag,
	253	unsigned int fromtag, unsigned int totag);
daff89f3	254	int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag);
e504f3fd	255	unsigned long radix_tree_locate_item(struct radix_tree_root root, void item);
1da177e4 LT	256
	257	static inline void radix_tree_preload_end(void)
	258	{
	259	preempt_enable();
	260	}
	261
	262	#endif /* _LINUX_RADIX_TREE_H */