[deliverable/linux.git] / include / linux / lockdep.h

/*
 * Runtime locking correctness validator
 *
 *  Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
 *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
 *
 * see Documentation/lockdep-design.txt for more details.
 */
#ifndef __LINUX_LOCKDEP_H
#define __LINUX_LOCKDEP_H

struct task_struct;
struct lockdep_map;

#ifdef CONFIG_LOCKDEP

#include <linux/linkage.h>
#include <linux/list.h>
#include <linux/debug_locks.h>
#include <linux/stacktrace.h>

/*
 * Lock-class usage-state bits:
 */
enum lock_usage_bit
{
	LOCK_USED = 0,
	LOCK_USED_IN_HARDIRQ,
	LOCK_USED_IN_SOFTIRQ,
	LOCK_ENABLED_SOFTIRQS,
	LOCK_ENABLED_HARDIRQS,
	LOCK_USED_IN_HARDIRQ_READ,
	LOCK_USED_IN_SOFTIRQ_READ,
	LOCK_ENABLED_SOFTIRQS_READ,
	LOCK_ENABLED_HARDIRQS_READ,
	LOCK_USAGE_STATES
};

/*
 * Usage-state bitmasks:
 */
#define LOCKF_USED			(1 << LOCK_USED)
#define LOCKF_USED_IN_HARDIRQ		(1 << LOCK_USED_IN_HARDIRQ)
#define LOCKF_USED_IN_SOFTIRQ		(1 << LOCK_USED_IN_SOFTIRQ)
#define LOCKF_ENABLED_HARDIRQS		(1 << LOCK_ENABLED_HARDIRQS)
#define LOCKF_ENABLED_SOFTIRQS		(1 << LOCK_ENABLED_SOFTIRQS)

#define LOCKF_ENABLED_IRQS (LOCKF_ENABLED_HARDIRQS | LOCKF_ENABLED_SOFTIRQS)
#define LOCKF_USED_IN_IRQ (LOCKF_USED_IN_HARDIRQ | LOCKF_USED_IN_SOFTIRQ)

#define LOCKF_USED_IN_HARDIRQ_READ	(1 << LOCK_USED_IN_HARDIRQ_READ)
#define LOCKF_USED_IN_SOFTIRQ_READ	(1 << LOCK_USED_IN_SOFTIRQ_READ)
#define LOCKF_ENABLED_HARDIRQS_READ	(1 << LOCK_ENABLED_HARDIRQS_READ)
#define LOCKF_ENABLED_SOFTIRQS_READ	(1 << LOCK_ENABLED_SOFTIRQS_READ)

#define LOCKF_ENABLED_IRQS_READ \
		(LOCKF_ENABLED_HARDIRQS_READ | LOCKF_ENABLED_SOFTIRQS_READ)
#define LOCKF_USED_IN_IRQ_READ \
		(LOCKF_USED_IN_HARDIRQ_READ | LOCKF_USED_IN_SOFTIRQ_READ)

#define MAX_LOCKDEP_SUBCLASSES		8UL

/*
 * Lock-classes are keyed via unique addresses, by embedding the
 * lockclass-key into the kernel (or module) .data section. (For
 * static locks we use the lock address itself as the key.)
 */
struct lockdep_subclass_key {
	char __one_byte;
} __attribute__ ((__packed__));

struct lock_class_key {
	struct lockdep_subclass_key	subkeys[MAX_LOCKDEP_SUBCLASSES];
};

#define LOCKSTAT_POINTS		4

/*
 * The lock-class itself:
 */
struct lock_class {
	/*
	 * class-hash:
	 */
	struct list_head		hash_entry;

	/*
	 * global list of all lock-classes:
	 */
	struct list_head		lock_entry;

	struct lockdep_subclass_key	*key;
	unsigned int			subclass;
	unsigned int			dep_gen_id;

	/*
	 * IRQ/softirq usage tracking bits:
	 */
	unsigned long			usage_mask;
	struct stack_trace		usage_traces[LOCK_USAGE_STATES];

	/*
	 * These fields represent a directed graph of lock dependencies,
	 * to every node we attach a list of "forward" and a list of
	 * "backward" graph nodes.
	 */
	struct list_head		locks_after, locks_before;

	/*
	 * Generation counter, when doing certain classes of graph walking,
	 * to ensure that we check one node only once:
	 */
	unsigned int			version;

	/*
	 * Statistics counter:
	 */
	unsigned long			ops;

	const char			*name;
	int				name_version;

#ifdef CONFIG_LOCK_STAT
	unsigned long			contention_point[LOCKSTAT_POINTS];
	unsigned long			contending_point[LOCKSTAT_POINTS];
#endif
};

#ifdef CONFIG_LOCK_STAT
struct lock_time {
	s64				min;
	s64				max;
	s64				total;
	unsigned long			nr;
};

enum bounce_type {
	bounce_acquired_write,
	bounce_acquired_read,
	bounce_contended_write,
	bounce_contended_read,
	nr_bounce_types,

	bounce_acquired = bounce_acquired_write,
	bounce_contended = bounce_contended_write,
};

struct lock_class_stats {
	unsigned long			contention_point[4];
	unsigned long			contending_point[4];
	struct lock_time		read_waittime;
	struct lock_time		write_waittime;
	struct lock_time		read_holdtime;
	struct lock_time		write_holdtime;
	unsigned long			bounces[nr_bounce_types];
};

struct lock_class_stats lock_stats(struct lock_class *class);
void clear_lock_stats(struct lock_class *class);
#endif

/*
 * Map the lock object (the lock instance) to the lock-class object.
 * This is embedded into specific lock instances:
 */
struct lockdep_map {
	struct lock_class_key		*key;
	struct lock_class		*class_cache;
	const char			*name;
#ifdef CONFIG_LOCK_STAT
	int				cpu;
	unsigned long			ip;
#endif
};

/*
 * Every lock has a list of other locks that were taken after it.
 * We only grow the list, never remove from it:
 */
struct lock_list {
	struct list_head		entry;
	struct lock_class		*class;
	struct stack_trace		trace;
	int				distance;
};

/*
 * We record lock dependency chains, so that we can cache them:
 */
struct lock_chain {
	u8				irq_context;
	u8				depth;
	u16				base;
	struct list_head		entry;
	u64				chain_key;
};

#define MAX_LOCKDEP_KEYS_BITS		13
/*
 * Subtract one because we offset hlock->class_idx by 1 in order
 * to make 0 mean no class. This avoids overflowing the class_idx
 * bitfield and hitting the BUG in hlock_class().
 */
#define MAX_LOCKDEP_KEYS		((1UL << MAX_LOCKDEP_KEYS_BITS) - 1)

struct held_lock {
	/*
	 * One-way hash of the dependency chain up to this point. We
	 * hash the hashes step by step as the dependency chain grows.
	 *
	 * We use it for dependency-caching and we skip detection
	 * passes and dependency-updates if there is a cache-hit, so
	 * it is absolutely critical for 100% coverage of the validator
	 * to have a unique key value for every unique dependency path
	 * that can occur in the system, to make a unique hash value
	 * as likely as possible - hence the 64-bit width.
	 *
	 * The task struct holds the current hash value (initialized
	 * with zero), here we store the previous hash value:
	 */
	u64				prev_chain_key;
	unsigned long			acquire_ip;
	struct lockdep_map		*instance;
	struct lockdep_map		*nest_lock;
#ifdef CONFIG_LOCK_STAT
	u64 				waittime_stamp;
	u64				holdtime_stamp;
#endif
	unsigned int			class_idx:MAX_LOCKDEP_KEYS_BITS;
	/*
	 * The lock-stack is unified in that the lock chains of interrupt
	 * contexts nest ontop of process context chains, but we 'separate'
	 * the hashes by starting with 0 if we cross into an interrupt
	 * context, and we also keep do not add cross-context lock
	 * dependencies - the lock usage graph walking covers that area
	 * anyway, and we'd just unnecessarily increase the number of
	 * dependencies otherwise. [Note: hardirq and softirq contexts
	 * are separated from each other too.]
	 *
	 * The following field is used to detect when we cross into an
	 * interrupt context:
	 */
	unsigned int irq_context:2; /* bit 0 - soft, bit 1 - hard */
	unsigned int trylock:1;
	unsigned int read:2;        /* see lock_acquire() comment */
	unsigned int check:2;       /* see lock_acquire() comment */
	unsigned int hardirqs_off:1;
};

/*
 * Initialization, self-test and debugging-output methods:
 */
extern void lockdep_init(void);
extern void lockdep_info(void);
extern void lockdep_reset(void);
extern void lockdep_reset_lock(struct lockdep_map *lock);
extern void lockdep_free_key_range(void *start, unsigned long size);
extern void lockdep_sys_exit(void);

extern void lockdep_off(void);
extern void lockdep_on(void);

/*
 * These methods are used by specific locking variants (spinlocks,
 * rwlocks, mutexes and rwsems) to pass init/acquire/release events
 * to lockdep:
 */

extern void lockdep_init_map(struct lockdep_map *lock, const char *name,
			     struct lock_class_key *key, int subclass);

/*
 * To initialize a lockdep_map statically use this macro.
 * Note that _name must not be NULL.
 */
#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
	{ .name = (_name), .key = (void *)(_key), }

/*
 * Reinitialize a lock key - for cases where there is special locking or
 * special initialization of locks so that the validator gets the scope
 * of dependencies wrong: they are either too broad (they need a class-split)
 * or they are too narrow (they suffer from a false class-split):
 */
#define lockdep_set_class(lock, key) \
		lockdep_init_map(&(lock)->dep_map, #key, key, 0)
#define lockdep_set_class_and_name(lock, key, name) \
		lockdep_init_map(&(lock)->dep_map, name, key, 0)
#define lockdep_set_class_and_subclass(lock, key, sub) \
		lockdep_init_map(&(lock)->dep_map, #key, key, sub)
#define lockdep_set_subclass(lock, sub)	\
		lockdep_init_map(&(lock)->dep_map, #lock, \
				 (lock)->dep_map.key, sub)

/*
 * Acquire a lock.
 *
 * Values for "read":
 *
 *   0: exclusive (write) acquire
 *   1: read-acquire (no recursion allowed)
 *   2: read-acquire with same-instance recursion allowed
 *
 * Values for check:
 *
 *   0: disabled
 *   1: simple checks (freeing, held-at-exit-time, etc.)
 *   2: full validation
 */
extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
			 int trylock, int read, int check,
			 struct lockdep_map *nest_lock, unsigned long ip);

extern void lock_release(struct lockdep_map *lock, int nested,
			 unsigned long ip);

extern void lock_set_subclass(struct lockdep_map *lock, unsigned int subclass,
			      unsigned long ip);

# define INIT_LOCKDEP				.lockdep_recursion = 0,

#define lockdep_depth(tsk)	(debug_locks ? (tsk)->lockdep_depth : 0)

#else /* !LOCKDEP */

static inline void lockdep_off(void)
{
}

static inline void lockdep_on(void)
{
}

# define lock_acquire(l, s, t, r, c, n, i)	do { } while (0)
# define lock_release(l, n, i)			do { } while (0)
# define lock_set_subclass(l, s, i)		do { } while (0)
# define lockdep_init()				do { } while (0)
# define lockdep_info()				do { } while (0)
# define lockdep_init_map(lock, name, key, sub)	do { (void)(key); } while (0)
# define lockdep_set_class(lock, key)		do { (void)(key); } while (0)
# define lockdep_set_class_and_name(lock, key, name) \
		do { (void)(key); } while (0)
#define lockdep_set_class_and_subclass(lock, key, sub) \
		do { (void)(key); } while (0)
#define lockdep_set_subclass(lock, sub)		do { } while (0)

# define INIT_LOCKDEP
# define lockdep_reset()		do { debug_locks = 1; } while (0)
# define lockdep_free_key_range(start, size)	do { } while (0)
# define lockdep_sys_exit() 			do { } while (0)
/*
 * The class key takes no space if lockdep is disabled:
 */
struct lock_class_key { };

#define lockdep_depth(tsk)	(0)

#endif /* !LOCKDEP */

#ifdef CONFIG_LOCK_STAT

extern void lock_contended(struct lockdep_map *lock, unsigned long ip);
extern void lock_acquired(struct lockdep_map *lock, unsigned long ip);

#define LOCK_CONTENDED(_lock, try, lock)			\
do {								\
	if (!try(_lock)) {					\
		lock_contended(&(_lock)->dep_map, _RET_IP_);	\
		lock(_lock);					\
	}							\
	lock_acquired(&(_lock)->dep_map, _RET_IP_);			\
} while (0)

#else /* CONFIG_LOCK_STAT */

#define lock_contended(lockdep_map, ip) do {} while (0)
#define lock_acquired(lockdep_map, ip) do {} while (0)

#define LOCK_CONTENDED(_lock, try, lock) \
	lock(_lock)

#endif /* CONFIG_LOCK_STAT */

#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_GENERIC_HARDIRQS)
extern void early_init_irq_lock_class(void);
#else
static inline void early_init_irq_lock_class(void)
{
}
#endif

#ifdef CONFIG_TRACE_IRQFLAGS
extern void early_boot_irqs_off(void);
extern void early_boot_irqs_on(void);
extern void print_irqtrace_events(struct task_struct *curr);
#else
static inline void early_boot_irqs_off(void)
{
}
static inline void early_boot_irqs_on(void)
{
}
static inline void print_irqtrace_events(struct task_struct *curr)
{
}
#endif

/*
 * For trivial one-depth nesting of a lock-class, the following
 * global define can be used. (Subsystems with multiple levels
 * of nesting should define their own lock-nesting subclasses.)
 */
#define SINGLE_DEPTH_NESTING			1

/*
 * Map the dependency ops to NOP or to real lockdep ops, depending
 * on the per lock-class debug mode:
 */

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define spin_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
#  define spin_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 2, n, i)
# else
#  define spin_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
#  define spin_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 1, NULL, i)
# endif
# define spin_release(l, n, i)			lock_release(l, n, i)
#else
# define spin_acquire(l, s, t, i)		do { } while (0)
# define spin_release(l, n, i)			do { } while (0)
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define rwlock_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
#  define rwlock_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 2, 2, NULL, i)
# else
#  define rwlock_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
#  define rwlock_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 2, 1, NULL, i)
# endif
# define rwlock_release(l, n, i)		lock_release(l, n, i)
#else
# define rwlock_acquire(l, s, t, i)		do { } while (0)
# define rwlock_acquire_read(l, s, t, i)	do { } while (0)
# define rwlock_release(l, n, i)		do { } while (0)
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define mutex_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
# else
#  define mutex_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
# endif
# define mutex_release(l, n, i)			lock_release(l, n, i)
#else
# define mutex_acquire(l, s, t, i)		do { } while (0)
# define mutex_release(l, n, i)			do { } while (0)
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define rwsem_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
#  define rwsem_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 1, 2, NULL, i)
# else
#  define rwsem_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
#  define rwsem_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 1, 1, NULL, i)
# endif
# define rwsem_release(l, n, i)			lock_release(l, n, i)
#else
# define rwsem_acquire(l, s, t, i)		do { } while (0)
# define rwsem_acquire_read(l, s, t, i)		do { } while (0)
# define rwsem_release(l, n, i)			do { } while (0)
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define lock_map_acquire(l)		lock_acquire(l, 0, 0, 0, 2, NULL, _THIS_IP_)
# else
#  define lock_map_acquire(l)		lock_acquire(l, 0, 0, 0, 1, NULL, _THIS_IP_)
# endif
# define lock_map_release(l)			lock_release(l, 1, _THIS_IP_)
#else
# define lock_map_acquire(l)			do { } while (0)
# define lock_map_release(l)			do { } while (0)
#endif

#ifdef CONFIG_PROVE_LOCKING
# define might_lock(lock) 						\
do {									\
	typecheck(struct lockdep_map *, &(lock)->dep_map);		\
	lock_acquire(&(lock)->dep_map, 0, 0, 0, 2, NULL, _THIS_IP_);	\
	lock_release(&(lock)->dep_map, 0, _THIS_IP_);			\
} while (0)
# define might_lock_read(lock) 						\
do {									\
	typecheck(struct lockdep_map *, &(lock)->dep_map);		\
	lock_acquire(&(lock)->dep_map, 0, 0, 1, 2, NULL, _THIS_IP_);	\
	lock_release(&(lock)->dep_map, 0, _THIS_IP_);			\
} while (0)
#else
# define might_lock(lock) do { } while (0)
# define might_lock_read(lock) do { } while (0)
#endif

#endif /* __LINUX_LOCKDEP_H */
Commit	Line	Data
fbb9ce95 IM	1	/*
	2	* Runtime locking correctness validator
	3	*
4b32d0a4 PZ	4	* Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
4b32d0a4 PZ	5	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
fbb9ce95 IM	6	*
	7	* see Documentation/lockdep-design.txt for more details.
	8	*/
	9	#ifndef __LINUX_LOCKDEP_H
	10	#define __LINUX_LOCKDEP_H
	11
a1e96b03	12	struct task_struct;
f20786ff	13	struct lockdep_map;
a1e96b03	14
db0b0ead MT	15	#ifdef CONFIG_LOCKDEP
db0b0ead MT	16
fbb9ce95 IM	17	#include <linux/linkage.h>
	18	#include <linux/list.h>
	19	#include <linux/debug_locks.h>
	20	#include <linux/stacktrace.h>
	21
fbb9ce95 IM	22	/*
	23	* Lock-class usage-state bits:
	24	*/
	25	enum lock_usage_bit
	26	{
	27	LOCK_USED = 0,
	28	LOCK_USED_IN_HARDIRQ,
	29	LOCK_USED_IN_SOFTIRQ,
	30	LOCK_ENABLED_SOFTIRQS,
	31	LOCK_ENABLED_HARDIRQS,
	32	LOCK_USED_IN_HARDIRQ_READ,
	33	LOCK_USED_IN_SOFTIRQ_READ,
	34	LOCK_ENABLED_SOFTIRQS_READ,
	35	LOCK_ENABLED_HARDIRQS_READ,
	36	LOCK_USAGE_STATES
	37	};
	38
	39	/*
	40	* Usage-state bitmasks:
	41	*/
	42	#define LOCKF_USED (1 << LOCK_USED)
	43	#define LOCKF_USED_IN_HARDIRQ (1 << LOCK_USED_IN_HARDIRQ)
	44	#define LOCKF_USED_IN_SOFTIRQ (1 << LOCK_USED_IN_SOFTIRQ)
	45	#define LOCKF_ENABLED_HARDIRQS (1 << LOCK_ENABLED_HARDIRQS)
	46	#define LOCKF_ENABLED_SOFTIRQS (1 << LOCK_ENABLED_SOFTIRQS)
	47
	48	#define LOCKF_ENABLED_IRQS (LOCKF_ENABLED_HARDIRQS \| LOCKF_ENABLED_SOFTIRQS)
	49	#define LOCKF_USED_IN_IRQ (LOCKF_USED_IN_HARDIRQ \| LOCKF_USED_IN_SOFTIRQ)
	50
	51	#define LOCKF_USED_IN_HARDIRQ_READ (1 << LOCK_USED_IN_HARDIRQ_READ)
	52	#define LOCKF_USED_IN_SOFTIRQ_READ (1 << LOCK_USED_IN_SOFTIRQ_READ)
	53	#define LOCKF_ENABLED_HARDIRQS_READ (1 << LOCK_ENABLED_HARDIRQS_READ)
	54	#define LOCKF_ENABLED_SOFTIRQS_READ (1 << LOCK_ENABLED_SOFTIRQS_READ)
	55
	56	#define LOCKF_ENABLED_IRQS_READ \
	57	(LOCKF_ENABLED_HARDIRQS_READ \| LOCKF_ENABLED_SOFTIRQS_READ)
	58	#define LOCKF_USED_IN_IRQ_READ \
	59	(LOCKF_USED_IN_HARDIRQ_READ \| LOCKF_USED_IN_SOFTIRQ_READ)
	60
	61	#define MAX_LOCKDEP_SUBCLASSES 8UL
	62
	63	/*
	64	* Lock-classes are keyed via unique addresses, by embedding the
	65	* lockclass-key into the kernel (or module) .data section. (For
	66	* static locks we use the lock address itself as the key.)
	67	*/
	68	struct lockdep_subclass_key {
	69	char __one_byte;
	70	} __attribute__ ((__packed__));
	71
	72	struct lock_class_key {
	73	struct lockdep_subclass_key subkeys[MAX_LOCKDEP_SUBCLASSES];
	74	};
	75
c7e78cff PZ	76	#define LOCKSTAT_POINTS 4
c7e78cff PZ	77
fbb9ce95 IM	78	/*
	79	* The lock-class itself:
	80	*/
	81	struct lock_class {
	82	/*
	83	* class-hash:
	84	*/
	85	struct list_head hash_entry;
	86
	87	/*
	88	* global list of all lock-classes:
	89	*/
	90	struct list_head lock_entry;
	91
	92	struct lockdep_subclass_key *key;
	93	unsigned int subclass;
419ca3f1	94	unsigned int dep_gen_id;
fbb9ce95 IM	95
	96	/*
	97	* IRQ/softirq usage tracking bits:
	98	*/
	99	unsigned long usage_mask;
	100	struct stack_trace usage_traces[LOCK_USAGE_STATES];
	101
	102	/*
	103	* These fields represent a directed graph of lock dependencies,
	104	* to every node we attach a list of "forward" and a list of
	105	* "backward" graph nodes.
	106	*/
	107	struct list_head locks_after, locks_before;
	108
	109	/*
	110	* Generation counter, when doing certain classes of graph walking,
	111	* to ensure that we check one node only once:
	112	*/
	113	unsigned int version;
	114
	115	/*
	116	* Statistics counter:
	117	*/
	118	unsigned long ops;
	119
	120	const char *name;
	121	int name_version;
f20786ff PZ	122
f20786ff PZ	123	#ifdef CONFIG_LOCK_STAT
c7e78cff PZ	124	unsigned long contention_point[LOCKSTAT_POINTS];
c7e78cff PZ	125	unsigned long contending_point[LOCKSTAT_POINTS];
f20786ff PZ	126	#endif
	127	};
	128
	129	#ifdef CONFIG_LOCK_STAT
	130	struct lock_time {
	131	s64 min;
	132	s64 max;
	133	s64 total;
	134	unsigned long nr;
fbb9ce95 IM	135	};
fbb9ce95 IM	136
96645678 PZ	137	enum bounce_type {
	138	bounce_acquired_write,
	139	bounce_acquired_read,
	140	bounce_contended_write,
	141	bounce_contended_read,
	142	nr_bounce_types,
	143
	144	bounce_acquired = bounce_acquired_write,
	145	bounce_contended = bounce_contended_write,
	146	};
	147
f20786ff PZ	148	struct lock_class_stats {
f20786ff PZ	149	unsigned long contention_point[4];
c7e78cff	150	unsigned long contending_point[4];
f20786ff PZ	151	struct lock_time read_waittime;
	152	struct lock_time write_waittime;
	153	struct lock_time read_holdtime;
	154	struct lock_time write_holdtime;
96645678	155	unsigned long bounces[nr_bounce_types];
f20786ff PZ	156	};
	157
	158	struct lock_class_stats lock_stats(struct lock_class *class);
	159	void clear_lock_stats(struct lock_class *class);
	160	#endif
	161
fbb9ce95 IM	162	/*
	163	* Map the lock object (the lock instance) to the lock-class object.
	164	* This is embedded into specific lock instances:
	165	*/
	166	struct lockdep_map {
	167	struct lock_class_key *key;
d6d897ce	168	struct lock_class *class_cache;
fbb9ce95	169	const char *name;
96645678 PZ	170	#ifdef CONFIG_LOCK_STAT
96645678 PZ	171	int cpu;
c7e78cff	172	unsigned long ip;
96645678	173	#endif
fbb9ce95 IM	174	};
	175
	176	/*
	177	* Every lock has a list of other locks that were taken after it.
	178	* We only grow the list, never remove from it:
	179	*/
	180	struct lock_list {
	181	struct list_head entry;
	182	struct lock_class *class;
	183	struct stack_trace trace;
068135e6	184	int distance;
fbb9ce95 IM	185	};
	186
	187	/*
	188	* We record lock dependency chains, so that we can cache them:
	189	*/
	190	struct lock_chain {
443cd507 HY	191	u8 irq_context;
	192	u8 depth;
	193	u16 base;
fbb9ce95 IM	194	struct list_head entry;
	195	u64 chain_key;
	196	};
	197
e5f363e3	198	#define MAX_LOCKDEP_KEYS_BITS 13
b42e737e PZ	199	/*
	200	* Subtract one because we offset hlock->class_idx by 1 in order
	201	* to make 0 mean no class. This avoids overflowing the class_idx
	202	* bitfield and hitting the BUG in hlock_class().
	203	*/
	204	#define MAX_LOCKDEP_KEYS ((1UL << MAX_LOCKDEP_KEYS_BITS) - 1)
f82b217e	205
fbb9ce95 IM	206	struct held_lock {
	207	/*
	208	* One-way hash of the dependency chain up to this point. We
	209	* hash the hashes step by step as the dependency chain grows.
	210	*
	211	* We use it for dependency-caching and we skip detection
	212	* passes and dependency-updates if there is a cache-hit, so
	213	* it is absolutely critical for 100% coverage of the validator
	214	* to have a unique key value for every unique dependency path
	215	* that can occur in the system, to make a unique hash value
	216	* as likely as possible - hence the 64-bit width.
	217	*
	218	* The task struct holds the current hash value (initialized
	219	* with zero), here we store the previous hash value:
	220	*/
	221	u64 prev_chain_key;
fbb9ce95 IM	222	unsigned long acquire_ip;
fbb9ce95 IM	223	struct lockdep_map *instance;
7531e2f3	224	struct lockdep_map *nest_lock;
f20786ff PZ	225	#ifdef CONFIG_LOCK_STAT
	226	u64 waittime_stamp;
	227	u64 holdtime_stamp;
	228	#endif
f82b217e	229	unsigned int class_idx:MAX_LOCKDEP_KEYS_BITS;
fbb9ce95 IM	230	/*
	231	* The lock-stack is unified in that the lock chains of interrupt
	232	* contexts nest ontop of process context chains, but we 'separate'
	233	* the hashes by starting with 0 if we cross into an interrupt
	234	* context, and we also keep do not add cross-context lock
	235	* dependencies - the lock usage graph walking covers that area
	236	* anyway, and we'd just unnecessarily increase the number of
	237	* dependencies otherwise. [Note: hardirq and softirq contexts
	238	* are separated from each other too.]
	239	*
	240	* The following field is used to detect when we cross into an
	241	* interrupt context:
	242	*/
f82b217e DJ	243	unsigned int irq_context:2; /* bit 0 - soft, bit 1 - hard */
	244	unsigned int trylock:1;
	245	unsigned int read:2; /* see lock_acquire() comment */
	246	unsigned int check:2; /* see lock_acquire() comment */
	247	unsigned int hardirqs_off:1;
fbb9ce95 IM	248	};
	249
	250	/*
	251	* Initialization, self-test and debugging-output methods:
	252	*/
	253	extern void lockdep_init(void);
	254	extern void lockdep_info(void);
	255	extern void lockdep_reset(void);
	256	extern void lockdep_reset_lock(struct lockdep_map *lock);
	257	extern void lockdep_free_key_range(void *start, unsigned long size);
b351d164	258	extern void lockdep_sys_exit(void);
fbb9ce95 IM	259
	260	extern void lockdep_off(void);
	261	extern void lockdep_on(void);
fbb9ce95 IM	262
	263	/*
	264	* These methods are used by specific locking variants (spinlocks,
	265	* rwlocks, mutexes and rwsems) to pass init/acquire/release events
	266	* to lockdep:
	267	*/
	268
	269	extern void lockdep_init_map(struct lockdep_map lock, const char name,
4dfbb9d8	270	struct lock_class_key *key, int subclass);
fbb9ce95	271
851a67b8 PZ	272	/*
	273	* To initialize a lockdep_map statically use this macro.
	274	* Note that _name must not be NULL.
	275	*/
	276	#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
	277	{ .name = (_name), .key = (void *)(_key), }
	278
fbb9ce95 IM	279	/*
	280	* Reinitialize a lock key - for cases where there is special locking or
	281	* special initialization of locks so that the validator gets the scope
	282	* of dependencies wrong: they are either too broad (they need a class-split)
	283	* or they are too narrow (they suffer from a false class-split):
	284	*/
	285	#define lockdep_set_class(lock, key) \
4dfbb9d8	286	lockdep_init_map(&(lock)->dep_map, #key, key, 0)
fbb9ce95	287	#define lockdep_set_class_and_name(lock, key, name) \
4dfbb9d8 PZ	288	lockdep_init_map(&(lock)->dep_map, name, key, 0)
	289	#define lockdep_set_class_and_subclass(lock, key, sub) \
	290	lockdep_init_map(&(lock)->dep_map, #key, key, sub)
	291	#define lockdep_set_subclass(lock, sub) \
	292	lockdep_init_map(&(lock)->dep_map, #lock, \
	293	(lock)->dep_map.key, sub)
fbb9ce95 IM	294
	295	/*
	296	* Acquire a lock.
	297	*
	298	* Values for "read":
	299	*
	300	* 0: exclusive (write) acquire
	301	* 1: read-acquire (no recursion allowed)
	302	* 2: read-acquire with same-instance recursion allowed
	303	*
	304	* Values for check:
	305	*
	306	* 0: disabled
	307	* 1: simple checks (freeing, held-at-exit-time, etc.)
	308	* 2: full validation
	309	*/
	310	extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
7531e2f3 PZ	311	int trylock, int read, int check,
7531e2f3 PZ	312	struct lockdep_map *nest_lock, unsigned long ip);
fbb9ce95 IM	313
	314	extern void lock_release(struct lockdep_map *lock, int nested,
	315	unsigned long ip);
	316
64aa348e PZ	317	extern void lock_set_subclass(struct lockdep_map *lock, unsigned int subclass,
	318	unsigned long ip);
	319
fbb9ce95 IM	320	# define INIT_LOCKDEP .lockdep_recursion = 0,
fbb9ce95 IM	321
e3a55fd1	322	#define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0)
d5abe669	323
fbb9ce95 IM	324	#else /* !LOCKDEP */
	325
	326	static inline void lockdep_off(void)
	327	{
	328	}
	329
	330	static inline void lockdep_on(void)
	331	{
	332	}
	333
7531e2f3	334	# define lock_acquire(l, s, t, r, c, n, i) do { } while (0)
fbb9ce95	335	# define lock_release(l, n, i) do { } while (0)
64aa348e	336	# define lock_set_subclass(l, s, i) do { } while (0)
fbb9ce95 IM	337	# define lockdep_init() do { } while (0)
fbb9ce95 IM	338	# define lockdep_info() do { } while (0)
4dfbb9d8	339	# define lockdep_init_map(lock, name, key, sub) do { (void)(key); } while (0)
fbb9ce95 IM	340	# define lockdep_set_class(lock, key) do { (void)(key); } while (0)
	341	# define lockdep_set_class_and_name(lock, key, name) \
	342	do { (void)(key); } while (0)
4dfbb9d8 PZ	343	#define lockdep_set_class_and_subclass(lock, key, sub) \
4dfbb9d8 PZ	344	do { (void)(key); } while (0)
07646e21 AM	345	#define lockdep_set_subclass(lock, sub) do { } while (0)
07646e21 AM	346
fbb9ce95 IM	347	# define INIT_LOCKDEP
	348	# define lockdep_reset() do { debug_locks = 1; } while (0)
	349	# define lockdep_free_key_range(start, size) do { } while (0)
b351d164	350	# define lockdep_sys_exit() do { } while (0)
fbb9ce95 IM	351	/*
	352	* The class key takes no space if lockdep is disabled:
	353	*/
	354	struct lock_class_key { };
d5abe669 PZ	355
	356	#define lockdep_depth(tsk) (0)
	357
fbb9ce95 IM	358	#endif /* !LOCKDEP */
fbb9ce95 IM	359
f20786ff PZ	360	#ifdef CONFIG_LOCK_STAT
	361
	362	extern void lock_contended(struct lockdep_map *lock, unsigned long ip);
c7e78cff	363	extern void lock_acquired(struct lockdep_map *lock, unsigned long ip);
f20786ff PZ	364
	365	#define LOCK_CONTENDED(_lock, try, lock) \
	366	do { \
	367	if (!try(_lock)) { \
	368	lock_contended(&(_lock)->dep_map, _RET_IP_); \
	369	lock(_lock); \
f20786ff	370	} \
c7e78cff	371	lock_acquired(&(_lock)->dep_map, _RET_IP_); \
f20786ff PZ	372	} while (0)
	373
	374	#else /* CONFIG_LOCK_STAT */
	375
	376	#define lock_contended(lockdep_map, ip) do {} while (0)
c7e78cff	377	#define lock_acquired(lockdep_map, ip) do {} while (0)
f20786ff PZ	378
	379	#define LOCK_CONTENDED(_lock, try, lock) \
	380	lock(_lock)
	381
	382	#endif /* CONFIG_LOCK_STAT */
	383
243c7621 IM	384	#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_GENERIC_HARDIRQS)
	385	extern void early_init_irq_lock_class(void);
	386	#else
3117df04 IM	387	static inline void early_init_irq_lock_class(void)
	388	{
	389	}
243c7621 IM	390	#endif
243c7621 IM	391
fbb9ce95 IM	392	#ifdef CONFIG_TRACE_IRQFLAGS
	393	extern void early_boot_irqs_off(void);
	394	extern void early_boot_irqs_on(void);
3117df04	395	extern void print_irqtrace_events(struct task_struct *curr);
fbb9ce95	396	#else
3117df04 IM	397	static inline void early_boot_irqs_off(void)
	398	{
	399	}
	400	static inline void early_boot_irqs_on(void)
	401	{
	402	}
	403	static inline void print_irqtrace_events(struct task_struct *curr)
	404	{
	405	}
fbb9ce95 IM	406	#endif
	407
	408	/*
	409	* For trivial one-depth nesting of a lock-class, the following
	410	* global define can be used. (Subsystems with multiple levels
	411	* of nesting should define their own lock-nesting subclasses.)
	412	*/
	413	#define SINGLE_DEPTH_NESTING 1
	414
	415	/*
	416	* Map the dependency ops to NOP or to real lockdep ops, depending
	417	* on the per lock-class debug mode:
	418	*/
	419
	420	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	421	# ifdef CONFIG_PROVE_LOCKING
7531e2f3	422	# define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
b7d39aff	423	# define spin_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 2, n, i)
fbb9ce95	424	# else
7531e2f3	425	# define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
b7d39aff	426	# define spin_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 1, NULL, i)
fbb9ce95 IM	427	# endif
	428	# define spin_release(l, n, i) lock_release(l, n, i)
	429	#else
	430	# define spin_acquire(l, s, t, i) do { } while (0)
	431	# define spin_release(l, n, i) do { } while (0)
	432	#endif
	433
	434	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	435	# ifdef CONFIG_PROVE_LOCKING
7531e2f3 PZ	436	# define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
7531e2f3 PZ	437	# define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 2, NULL, i)
fbb9ce95	438	# else
7531e2f3 PZ	439	# define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
7531e2f3 PZ	440	# define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 1, NULL, i)
fbb9ce95 IM	441	# endif
	442	# define rwlock_release(l, n, i) lock_release(l, n, i)
	443	#else
	444	# define rwlock_acquire(l, s, t, i) do { } while (0)
	445	# define rwlock_acquire_read(l, s, t, i) do { } while (0)
	446	# define rwlock_release(l, n, i) do { } while (0)
	447	#endif
	448
	449	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	450	# ifdef CONFIG_PROVE_LOCKING
7531e2f3	451	# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
fbb9ce95	452	# else
7531e2f3	453	# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
fbb9ce95 IM	454	# endif
	455	# define mutex_release(l, n, i) lock_release(l, n, i)
	456	#else
	457	# define mutex_acquire(l, s, t, i) do { } while (0)
	458	# define mutex_release(l, n, i) do { } while (0)
	459	#endif
	460
	461	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	462	# ifdef CONFIG_PROVE_LOCKING
7531e2f3 PZ	463	# define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
7531e2f3 PZ	464	# define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 2, NULL, i)
fbb9ce95	465	# else
7531e2f3 PZ	466	# define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
7531e2f3 PZ	467	# define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 1, NULL, i)
fbb9ce95 IM	468	# endif
	469	# define rwsem_release(l, n, i) lock_release(l, n, i)
	470	#else
	471	# define rwsem_acquire(l, s, t, i) do { } while (0)
	472	# define rwsem_acquire_read(l, s, t, i) do { } while (0)
	473	# define rwsem_release(l, n, i) do { } while (0)
	474	#endif
	475
4f3e7524 PZ	476	#ifdef CONFIG_DEBUG_LOCK_ALLOC
4f3e7524 PZ	477	# ifdef CONFIG_PROVE_LOCKING
3295f0ef	478	# define lock_map_acquire(l) lock_acquire(l, 0, 0, 0, 2, NULL, _THIS_IP_)
4f3e7524	479	# else
3295f0ef	480	# define lock_map_acquire(l) lock_acquire(l, 0, 0, 0, 1, NULL, _THIS_IP_)
4f3e7524	481	# endif
3295f0ef	482	# define lock_map_release(l) lock_release(l, 1, _THIS_IP_)
4f3e7524	483	#else
3295f0ef IM	484	# define lock_map_acquire(l) do { } while (0)
3295f0ef IM	485	# define lock_map_release(l) do { } while (0)
4f3e7524 PZ	486	#endif
4f3e7524 PZ	487
76b189e9 PZ	488	#ifdef CONFIG_PROVE_LOCKING
	489	# define might_lock(lock) \
	490	do { \
	491	typecheck(struct lockdep_map *, &(lock)->dep_map); \
	492	lock_acquire(&(lock)->dep_map, 0, 0, 0, 2, NULL, _THIS_IP_); \
	493	lock_release(&(lock)->dep_map, 0, _THIS_IP_); \
	494	} while (0)
	495	# define might_lock_read(lock) \
	496	do { \
	497	typecheck(struct lockdep_map *, &(lock)->dep_map); \
	498	lock_acquire(&(lock)->dep_map, 0, 0, 1, 2, NULL, _THIS_IP_); \
	499	lock_release(&(lock)->dep_map, 0, _THIS_IP_); \
	500	} while (0)
	501	#else
	502	# define might_lock(lock) do { } while (0)
	503	# define might_lock_read(lock) do { } while (0)
	504	#endif
	505
fbb9ce95	506	#endif /* __LINUX_LOCKDEP_H */