[deliverable/linux.git] / arch / x86 / kernel / hw_breakpoint.c

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) 2007 Alan Stern
 * Copyright (C) 2009 IBM Corporation
 * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com>
 *
 * Authors: Alan Stern <stern@rowland.harvard.edu>
 *          K.Prasad <prasad@linux.vnet.ibm.com>
 *          Frederic Weisbecker <fweisbec@gmail.com>
 */

/*
 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
 * using the CPU's debug registers.
 */

#include <linux/perf_event.h>
#include <linux/hw_breakpoint.h>
#include <linux/irqflags.h>
#include <linux/notifier.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/percpu.h>
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/smp.h>

#include <asm/hw_breakpoint.h>
#include <asm/processor.h>
#include <asm/debugreg.h>

/* Per cpu debug control register value */
DEFINE_PER_CPU(unsigned long, cpu_dr7);
EXPORT_PER_CPU_SYMBOL(cpu_dr7);

/* Per cpu debug address registers values */
static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]);

/*
 * Stores the breakpoints currently in use on each breakpoint address
 * register for each cpus
 */
static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]);


static inline unsigned long
__encode_dr7(int drnum, unsigned int len, unsigned int type)
{
	unsigned long bp_info;

	bp_info = (len | type) & 0xf;
	bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE);
	bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE));

	return bp_info;
}

/*
 * Encode the length, type, Exact, and Enable bits for a particular breakpoint
 * as stored in debug register 7.
 */
unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type)
{
	return __encode_dr7(drnum, len, type) | DR_GLOBAL_SLOWDOWN;
}

/*
 * Decode the length and type bits for a particular breakpoint as
 * stored in debug register 7.  Return the "enabled" status.
 */
int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type)
{
	int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE);

	*len = (bp_info & 0xc) | 0x40;
	*type = (bp_info & 0x3) | 0x80;

	return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3;
}

/*
 * Install a perf counter breakpoint.
 *
 * We seek a free debug address register and use it for this
 * breakpoint. Eventually we enable it in the debug control register.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
 */
int arch_install_hw_breakpoint(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned long *dr7;
	int i;

	for (i = 0; i < HBP_NUM; i++) {
		struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]);

		if (!*slot) {
			*slot = bp;
			break;
		}
	}

	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
		return -EBUSY;

	set_debugreg(info->address, i);
	__this_cpu_write(cpu_debugreg[i], info->address);

	dr7 = this_cpu_ptr(&cpu_dr7);
	*dr7 |= encode_dr7(i, info->len, info->type);

	set_debugreg(*dr7, 7);
	if (info->mask)
		set_dr_addr_mask(info->mask, i);

	return 0;
}

/*
 * Uninstall the breakpoint contained in the given counter.
 *
 * First we search the debug address register it uses and then we disable
 * it.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
 */
void arch_uninstall_hw_breakpoint(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned long *dr7;
	int i;

	for (i = 0; i < HBP_NUM; i++) {
		struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]);

		if (*slot == bp) {
			*slot = NULL;
			break;
		}
	}

	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
		return;

	dr7 = this_cpu_ptr(&cpu_dr7);
	*dr7 &= ~__encode_dr7(i, info->len, info->type);

	set_debugreg(*dr7, 7);
	if (info->mask)
		set_dr_addr_mask(0, i);
}

/*
 * Check for virtual address in kernel space.
 */
int arch_check_bp_in_kernelspace(struct perf_event *bp)
{
	unsigned int len;
	unsigned long va;
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);

	va = info->address;
	len = bp->attr.bp_len;

	/*
	 * We don't need to worry about va + len - 1 overflowing:
	 * we already require that va is aligned to a multiple of len.
	 */
	return (va >= TASK_SIZE_MAX) || ((va + len - 1) >= TASK_SIZE_MAX);
}

int arch_bp_generic_fields(int x86_len, int x86_type,
			   int *gen_len, int *gen_type)
{
	/* Type */
	switch (x86_type) {
	case X86_BREAKPOINT_EXECUTE:
		if (x86_len != X86_BREAKPOINT_LEN_X)
			return -EINVAL;

		*gen_type = HW_BREAKPOINT_X;
		*gen_len = sizeof(long);
		return 0;
	case X86_BREAKPOINT_WRITE:
		*gen_type = HW_BREAKPOINT_W;
		break;
	case X86_BREAKPOINT_RW:
		*gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R;
		break;
	default:
		return -EINVAL;
	}

	/* Len */
	switch (x86_len) {
	case X86_BREAKPOINT_LEN_1:
		*gen_len = HW_BREAKPOINT_LEN_1;
		break;
	case X86_BREAKPOINT_LEN_2:
		*gen_len = HW_BREAKPOINT_LEN_2;
		break;
	case X86_BREAKPOINT_LEN_4:
		*gen_len = HW_BREAKPOINT_LEN_4;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
		*gen_len = HW_BREAKPOINT_LEN_8;
		break;
#endif
	default:
		return -EINVAL;
	}

	return 0;
}


static int arch_build_bp_info(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);

	info->address = bp->attr.bp_addr;

	/* Type */
	switch (bp->attr.bp_type) {
	case HW_BREAKPOINT_W:
		info->type = X86_BREAKPOINT_WRITE;
		break;
	case HW_BREAKPOINT_W | HW_BREAKPOINT_R:
		info->type = X86_BREAKPOINT_RW;
		break;
	case HW_BREAKPOINT_X:
		/*
		 * We don't allow kernel breakpoints in places that are not
		 * acceptable for kprobes.  On non-kprobes kernels, we don't
		 * allow kernel breakpoints at all.
		 */
		if (bp->attr.bp_addr >= TASK_SIZE_MAX) {
#ifdef CONFIG_KPROBES
			if (within_kprobe_blacklist(bp->attr.bp_addr))
				return -EINVAL;
#else
			return -EINVAL;
#endif
		}

		info->type = X86_BREAKPOINT_EXECUTE;
		/*
		 * x86 inst breakpoints need to have a specific undefined len.
		 * But we still need to check userspace is not trying to setup
		 * an unsupported length, to get a range breakpoint for example.
		 */
		if (bp->attr.bp_len == sizeof(long)) {
			info->len = X86_BREAKPOINT_LEN_X;
			return 0;
		}
	default:
		return -EINVAL;
	}

	/* Len */
	info->mask = 0;

	switch (bp->attr.bp_len) {
	case HW_BREAKPOINT_LEN_1:
		info->len = X86_BREAKPOINT_LEN_1;
		break;
	case HW_BREAKPOINT_LEN_2:
		info->len = X86_BREAKPOINT_LEN_2;
		break;
	case HW_BREAKPOINT_LEN_4:
		info->len = X86_BREAKPOINT_LEN_4;
		break;
#ifdef CONFIG_X86_64
	case HW_BREAKPOINT_LEN_8:
		info->len = X86_BREAKPOINT_LEN_8;
		break;
#endif
	default:
		/* AMD range breakpoint */
		if (!is_power_of_2(bp->attr.bp_len))
			return -EINVAL;
		if (bp->attr.bp_addr & (bp->attr.bp_len - 1))
			return -EINVAL;

		if (!boot_cpu_has(X86_FEATURE_BPEXT))
			return -EOPNOTSUPP;

		/*
		 * It's impossible to use a range breakpoint to fake out
		 * user vs kernel detection because bp_len - 1 can't
		 * have the high bit set.  If we ever allow range instruction
		 * breakpoints, then we'll have to check for kprobe-blacklisted
		 * addresses anywhere in the range.
		 */
		info->mask = bp->attr.bp_len - 1;
		info->len = X86_BREAKPOINT_LEN_1;
	}

	return 0;
}

/*
 * Validate the arch-specific HW Breakpoint register settings
 */
int arch_validate_hwbkpt_settings(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned int align;
	int ret;


	ret = arch_build_bp_info(bp);
	if (ret)
		return ret;

	switch (info->len) {
	case X86_BREAKPOINT_LEN_1:
		align = 0;
		if (info->mask)
			align = info->mask;
		break;
	case X86_BREAKPOINT_LEN_2:
		align = 1;
		break;
	case X86_BREAKPOINT_LEN_4:
		align = 3;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
		align = 7;
		break;
#endif
	default:
		WARN_ON_ONCE(1);
	}

	/*
	 * Check that the low-order bits of the address are appropriate
	 * for the alignment implied by len.
	 */
	if (info->address & align)
		return -EINVAL;

	return 0;
}

/*
 * Dump the debug register contents to the user.
 * We can't dump our per cpu values because it
 * may contain cpu wide breakpoint, something that
 * doesn't belong to the current task.
 *
 * TODO: include non-ptrace user breakpoints (perf)
 */
void aout_dump_debugregs(struct user *dump)
{
	int i;
	int dr7 = 0;
	struct perf_event *bp;
	struct arch_hw_breakpoint *info;
	struct thread_struct *thread = &current->thread;

	for (i = 0; i < HBP_NUM; i++) {
		bp = thread->ptrace_bps[i];

		if (bp && !bp->attr.disabled) {
			dump->u_debugreg[i] = bp->attr.bp_addr;
			info = counter_arch_bp(bp);
			dr7 |= encode_dr7(i, info->len, info->type);
		} else {
			dump->u_debugreg[i] = 0;
		}
	}

	dump->u_debugreg[4] = 0;
	dump->u_debugreg[5] = 0;
	dump->u_debugreg[6] = current->thread.debugreg6;

	dump->u_debugreg[7] = dr7;
}
EXPORT_SYMBOL_GPL(aout_dump_debugregs);

/*
 * Release the user breakpoints used by ptrace
 */
void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
{
	int i;
	struct thread_struct *t = &tsk->thread;

	for (i = 0; i < HBP_NUM; i++) {
		unregister_hw_breakpoint(t->ptrace_bps[i]);
		t->ptrace_bps[i] = NULL;
	}

	t->debugreg6 = 0;
	t->ptrace_dr7 = 0;
}

void hw_breakpoint_restore(void)
{
	set_debugreg(__this_cpu_read(cpu_debugreg[0]), 0);
	set_debugreg(__this_cpu_read(cpu_debugreg[1]), 1);
	set_debugreg(__this_cpu_read(cpu_debugreg[2]), 2);
	set_debugreg(__this_cpu_read(cpu_debugreg[3]), 3);
	set_debugreg(current->thread.debugreg6, 6);
	set_debugreg(__this_cpu_read(cpu_dr7), 7);
}
EXPORT_SYMBOL_GPL(hw_breakpoint_restore);

/*
 * Handle debug exception notifications.
 *
 * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
 *
 * NOTIFY_DONE returned if one of the following conditions is true.
 * i) When the causative address is from user-space and the exception
 * is a valid one, i.e. not triggered as a result of lazy debug register
 * switching
 * ii) When there are more bits than trap<n> set in DR6 register (such
 * as BD, BS or BT) indicating that more than one debug condition is
 * met and requires some more action in do_debug().
 *
 * NOTIFY_STOP returned for all other cases
 *
 */
static int hw_breakpoint_handler(struct die_args *args)
{
	int i, cpu, rc = NOTIFY_STOP;
	struct perf_event *bp;
	unsigned long dr7, dr6;
	unsigned long *dr6_p;

	/* The DR6 value is pointed by args->err */
	dr6_p = (unsigned long *)ERR_PTR(args->err);
	dr6 = *dr6_p;

	/* If it's a single step, TRAP bits are random */
	if (dr6 & DR_STEP)
		return NOTIFY_DONE;

	/* Do an early return if no trap bits are set in DR6 */
	if ((dr6 & DR_TRAP_BITS) == 0)
		return NOTIFY_DONE;

	get_debugreg(dr7, 7);
	/* Disable breakpoints during exception handling */
	set_debugreg(0UL, 7);
	/*
	 * Assert that local interrupts are disabled
	 * Reset the DRn bits in the virtualized register value.
	 * The ptrace trigger routine will add in whatever is needed.
	 */
	current->thread.debugreg6 &= ~DR_TRAP_BITS;
	cpu = get_cpu();

	/* Handle all the breakpoints that were triggered */
	for (i = 0; i < HBP_NUM; ++i) {
		if (likely(!(dr6 & (DR_TRAP0 << i))))
			continue;

		/*
		 * The counter may be concurrently released but that can only
		 * occur from a call_rcu() path. We can then safely fetch
		 * the breakpoint, use its callback, touch its counter
		 * while we are in an rcu_read_lock() path.
		 */
		rcu_read_lock();

		bp = per_cpu(bp_per_reg[i], cpu);
		/*
		 * Reset the 'i'th TRAP bit in dr6 to denote completion of
		 * exception handling
		 */
		(*dr6_p) &= ~(DR_TRAP0 << i);
		/*
		 * bp can be NULL due to lazy debug register switching
		 * or due to concurrent perf counter removing.
		 */
		if (!bp) {
			rcu_read_unlock();
			break;
		}

		perf_bp_event(bp, args->regs);

		/*
		 * Set up resume flag to avoid breakpoint recursion when
		 * returning back to origin.
		 */
		if (bp->hw.info.type == X86_BREAKPOINT_EXECUTE)
			args->regs->flags |= X86_EFLAGS_RF;

		rcu_read_unlock();
	}
	/*
	 * Further processing in do_debug() is needed for a) user-space
	 * breakpoints (to generate signals) and b) when the system has
	 * taken exception due to multiple causes
	 */
	if ((current->thread.debugreg6 & DR_TRAP_BITS) ||
	    (dr6 & (~DR_TRAP_BITS)))
		rc = NOTIFY_DONE;

	set_debugreg(dr7, 7);
	put_cpu();

	return rc;
}

/*
 * Handle debug exception notifications.
 */
int hw_breakpoint_exceptions_notify(
		struct notifier_block *unused, unsigned long val, void *data)
{
	if (val != DIE_DEBUG)
		return NOTIFY_DONE;

	return hw_breakpoint_handler(data);
}

void hw_breakpoint_pmu_read(struct perf_event *bp)
{
	/* TODO */
}
Commit	Line	Data
0067f129 P	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License as published by
	4	* the Free Software Foundation; either version 2 of the License, or
	5	* (at your option) any later version.
	6	*
	7	* This program is distributed in the hope that it will be useful,
	8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	10	* GNU General Public License for more details.
	11	*
	12	* You should have received a copy of the GNU General Public License
	13	* along with this program; if not, write to the Free Software
	14	* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	15	*
	16	* Copyright (C) 2007 Alan Stern
	17	* Copyright (C) 2009 IBM Corporation
24f1e32c	18	* Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com>
ba6909b7 P	19	*
	20	* Authors: Alan Stern <stern@rowland.harvard.edu>
	21	* K.Prasad <prasad@linux.vnet.ibm.com>
	22	* Frederic Weisbecker <fweisbec@gmail.com>
0067f129 P	23	*/
	24
	25	/*
	26	* HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
	27	* using the CPU's debug registers.
	28	*/
	29
24f1e32c FW	30	#include <linux/perf_event.h>
24f1e32c FW	31	#include <linux/hw_breakpoint.h>
0067f129 P	32	#include <linux/irqflags.h>
	33	#include <linux/notifier.h>
	34	#include <linux/kallsyms.h>
e5779e8e	35	#include <linux/kprobes.h>
0067f129 P	36	#include <linux/percpu.h>
	37	#include <linux/kdebug.h>
	38	#include <linux/kernel.h>
	39	#include <linux/module.h>
	40	#include <linux/sched.h>
0067f129 P	41	#include <linux/smp.h>
	42
	43	#include <asm/hw_breakpoint.h>
	44	#include <asm/processor.h>
	45	#include <asm/debugreg.h>
	46
24f1e32c	47	/* Per cpu debug control register value */
28b4e0d8 TH	48	DEFINE_PER_CPU(unsigned long, cpu_dr7);
28b4e0d8 TH	49	EXPORT_PER_CPU_SYMBOL(cpu_dr7);
24f1e32c FW	50
	51	/* Per cpu debug address registers values */
	52	static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]);
0067f129 P	53
0067f129 P	54	/*
24f1e32c FW	55	* Stores the breakpoints currently in use on each breakpoint address
24f1e32c FW	56	* register for each cpus
0067f129	57	*/
24f1e32c	58	static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]);
0067f129 P	59
0067f129 P	60
2c31b795 FW	61	static inline unsigned long
2c31b795 FW	62	__encode_dr7(int drnum, unsigned int len, unsigned int type)
0067f129 P	63	{
	64	unsigned long bp_info;
	65
	66	bp_info = (len \| type) & 0xf;
	67	bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE);
2c31b795 FW	68	bp_info \|= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE));
2c31b795 FW	69
0067f129 P	70	return bp_info;
	71	}
	72
2c31b795 FW	73	/*
	74	* Encode the length, type, Exact, and Enable bits for a particular breakpoint
	75	* as stored in debug register 7.
	76	*/
	77	unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type)
	78	{
	79	return __encode_dr7(drnum, len, type) \| DR_GLOBAL_SLOWDOWN;
	80	}
	81
24f1e32c FW	82	/*
	83	* Decode the length and type bits for a particular breakpoint as
	84	* stored in debug register 7. Return the "enabled" status.
	85	*/
	86	int decode_dr7(unsigned long dr7, int bpnum, unsigned len, unsigned type)
0067f129	87	{
24f1e32c	88	int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE);
0067f129	89
24f1e32c FW	90	*len = (bp_info & 0xc) \| 0x40;
24f1e32c FW	91	*type = (bp_info & 0x3) \| 0x80;
0067f129	92
24f1e32c	93	return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3;
0067f129 P	94	}
	95
	96	/*
24f1e32c FW	97	* Install a perf counter breakpoint.
	98	*
	99	* We seek a free debug address register and use it for this
	100	* breakpoint. Eventually we enable it in the debug control register.
	101	*
	102	* Atomic: we hold the counter->ctx->lock and we only handle variables
	103	* and registers local to this cpu.
0067f129	104	*/
24f1e32c	105	int arch_install_hw_breakpoint(struct perf_event *bp)
0067f129	106	{
24f1e32c FW	107	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	108	unsigned long *dr7;
	109	int i;
	110
	111	for (i = 0; i < HBP_NUM; i++) {
89cbc767	112	struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]);
24f1e32c FW	113
	114	if (!*slot) {
	115	*slot = bp;
	116	break;
	117	}
0067f129 P	118	}
0067f129 P	119
24f1e32c FW	120	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
	121	return -EBUSY;
	122
	123	set_debugreg(info->address, i);
0a3aee0d	124	__this_cpu_write(cpu_debugreg[i], info->address);
24f1e32c	125
89cbc767	126	dr7 = this_cpu_ptr(&cpu_dr7);
24f1e32c FW	127	*dr7 \|= encode_dr7(i, info->len, info->type);
	128
	129	set_debugreg(*dr7, 7);
d6d55f0b JS	130	if (info->mask)
d6d55f0b JS	131	set_dr_addr_mask(info->mask, i);
24f1e32c FW	132
24f1e32c FW	133	return 0;
0067f129 P	134	}
	135
	136	/*
24f1e32c FW	137	* Uninstall the breakpoint contained in the given counter.
	138	*
	139	* First we search the debug address register it uses and then we disable
	140	* it.
	141	*
	142	* Atomic: we hold the counter->ctx->lock and we only handle variables
	143	* and registers local to this cpu.
0067f129	144	*/
24f1e32c	145	void arch_uninstall_hw_breakpoint(struct perf_event *bp)
0067f129	146	{
24f1e32c FW	147	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	148	unsigned long *dr7;
	149	int i;
	150
	151	for (i = 0; i < HBP_NUM; i++) {
89cbc767	152	struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]);
24f1e32c FW	153
	154	if (*slot == bp) {
	155	*slot = NULL;
	156	break;
	157	}
	158	}
	159
	160	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
	161	return;
0067f129	162
89cbc767	163	dr7 = this_cpu_ptr(&cpu_dr7);
2c31b795	164	*dr7 &= ~__encode_dr7(i, info->len, info->type);
24f1e32c FW	165
24f1e32c FW	166	set_debugreg(*dr7, 7);
d6d55f0b JS	167	if (info->mask)
d6d55f0b JS	168	set_dr_addr_mask(0, i);
0067f129 P	169	}
0067f129 P	170
0067f129 P	171	/*
	172	* Check for virtual address in kernel space.
	173	*/
b2812d03	174	int arch_check_bp_in_kernelspace(struct perf_event *bp)
0067f129 P	175	{
0067f129 P	176	unsigned int len;
b2812d03 FW	177	unsigned long va;
b2812d03 FW	178	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
0067f129	179
b2812d03	180	va = info->address;
36748b95	181	len = bp->attr.bp_len;
0067f129	182
27747f8b AL	183	/*
	184	* We don't need to worry about va + len - 1 overflowing:
	185	* we already require that va is aligned to a multiple of len.
	186	*/
	187	return (va >= TASK_SIZE_MAX) \|\| ((va + len - 1) >= TASK_SIZE_MAX);
0067f129 P	188	}
0067f129 P	189
24f1e32c FW	190	int arch_bp_generic_fields(int x86_len, int x86_type,
24f1e32c FW	191	int gen_len, int gen_type)
0067f129	192	{
89e45aac FW	193	/* Type */
	194	switch (x86_type) {
	195	case X86_BREAKPOINT_EXECUTE:
	196	if (x86_len != X86_BREAKPOINT_LEN_X)
	197	return -EINVAL;
	198
	199	*gen_type = HW_BREAKPOINT_X;
f7809daf	200	*gen_len = sizeof(long);
89e45aac FW	201	return 0;
	202	case X86_BREAKPOINT_WRITE:
	203	*gen_type = HW_BREAKPOINT_W;
f7809daf	204	break;
89e45aac FW	205	case X86_BREAKPOINT_RW:
	206	*gen_type = HW_BREAKPOINT_W \| HW_BREAKPOINT_R;
	207	break;
	208	default:
	209	return -EINVAL;
	210	}
	211
	212	/* Len */
	213	switch (x86_len) {
24f1e32c FW	214	case X86_BREAKPOINT_LEN_1:
	215	*gen_len = HW_BREAKPOINT_LEN_1;
	216	break;
	217	case X86_BREAKPOINT_LEN_2:
	218	*gen_len = HW_BREAKPOINT_LEN_2;
	219	break;
	220	case X86_BREAKPOINT_LEN_4:
	221	*gen_len = HW_BREAKPOINT_LEN_4;
	222	break;
	223	#ifdef CONFIG_X86_64
	224	case X86_BREAKPOINT_LEN_8:
	225	*gen_len = HW_BREAKPOINT_LEN_8;
	226	break;
	227	#endif
	228	default:
	229	return -EINVAL;
	230	}
0067f129	231
24f1e32c FW	232	return 0;
	233	}
	234
	235
	236	static int arch_build_bp_info(struct perf_event *bp)
	237	{
	238	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	239
	240	info->address = bp->attr.bp_addr;
	241
f7809daf FW	242	/* Type */
	243	switch (bp->attr.bp_type) {
	244	case HW_BREAKPOINT_W:
	245	info->type = X86_BREAKPOINT_WRITE;
	246	break;
	247	case HW_BREAKPOINT_W \| HW_BREAKPOINT_R:
	248	info->type = X86_BREAKPOINT_RW;
	249	break;
	250	case HW_BREAKPOINT_X:
e5779e8e AL	251	/*
	252	* We don't allow kernel breakpoints in places that are not
	253	* acceptable for kprobes. On non-kprobes kernels, we don't
	254	* allow kernel breakpoints at all.
	255	*/
	256	if (bp->attr.bp_addr >= TASK_SIZE_MAX) {
	257	#ifdef CONFIG_KPROBES
	258	if (within_kprobe_blacklist(bp->attr.bp_addr))
	259	return -EINVAL;
	260	#else
	261	return -EINVAL;
	262	#endif
	263	}
	264
f7809daf FW	265	info->type = X86_BREAKPOINT_EXECUTE;
	266	/*
	267	* x86 inst breakpoints need to have a specific undefined len.
	268	* But we still need to check userspace is not trying to setup
	269	* an unsupported length, to get a range breakpoint for example.
	270	*/
	271	if (bp->attr.bp_len == sizeof(long)) {
	272	info->len = X86_BREAKPOINT_LEN_X;
	273	return 0;
	274	}
	275	default:
	276	return -EINVAL;
	277	}
	278
24f1e32c	279	/* Len */
d6d55f0b JS	280	info->mask = 0;
d6d55f0b JS	281
24f1e32c	282	switch (bp->attr.bp_len) {
0067f129	283	case HW_BREAKPOINT_LEN_1:
24f1e32c	284	info->len = X86_BREAKPOINT_LEN_1;
0067f129 P	285	break;
0067f129 P	286	case HW_BREAKPOINT_LEN_2:
24f1e32c	287	info->len = X86_BREAKPOINT_LEN_2;
0067f129 P	288	break;
0067f129 P	289	case HW_BREAKPOINT_LEN_4:
24f1e32c	290	info->len = X86_BREAKPOINT_LEN_4;
0067f129 P	291	break;
	292	#ifdef CONFIG_X86_64
	293	case HW_BREAKPOINT_LEN_8:
24f1e32c FW	294	info->len = X86_BREAKPOINT_LEN_8;
	295	break;
	296	#endif
	297	default:
ab513927	298	/* AMD range breakpoint */
d6d55f0b JS	299	if (!is_power_of_2(bp->attr.bp_len))
d6d55f0b JS	300	return -EINVAL;
ab513927 AL	301	if (bp->attr.bp_addr & (bp->attr.bp_len - 1))
ab513927 AL	302	return -EINVAL;
362f924b BP	303
	304	if (!boot_cpu_has(X86_FEATURE_BPEXT))
	305	return -EOPNOTSUPP;
	306
ab513927 AL	307	/*
	308	* It's impossible to use a range breakpoint to fake out
	309	* user vs kernel detection because bp_len - 1 can't
	310	* have the high bit set. If we ever allow range instruction
	311	* breakpoints, then we'll have to check for kprobe-blacklisted
	312	* addresses anywhere in the range.
	313	*/
d6d55f0b JS	314	info->mask = bp->attr.bp_len - 1;
d6d55f0b JS	315	info->len = X86_BREAKPOINT_LEN_1;
24f1e32c FW	316	}
24f1e32c FW	317
24f1e32c FW	318	return 0;
24f1e32c FW	319	}
d6d55f0b	320
24f1e32c FW	321	/*
	322	* Validate the arch-specific HW Breakpoint register settings
	323	*/
b2812d03	324	int arch_validate_hwbkpt_settings(struct perf_event *bp)
24f1e32c FW	325	{
	326	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	327	unsigned int align;
	328	int ret;
	329
	330
	331	ret = arch_build_bp_info(bp);
	332	if (ret)
	333	return ret;
	334
24f1e32c FW	335	switch (info->len) {
	336	case X86_BREAKPOINT_LEN_1:
	337	align = 0;
d6d55f0b JS	338	if (info->mask)
d6d55f0b JS	339	align = info->mask;
24f1e32c FW	340	break;
	341	case X86_BREAKPOINT_LEN_2:
	342	align = 1;
	343	break;
	344	case X86_BREAKPOINT_LEN_4:
	345	align = 3;
	346	break;
	347	#ifdef CONFIG_X86_64
	348	case X86_BREAKPOINT_LEN_8:
0067f129 P	349	align = 7;
	350	break;
	351	#endif
	352	default:
d6d55f0b	353	WARN_ON_ONCE(1);
0067f129 P	354	}
0067f129 P	355
0067f129 P	356	/*
	357	* Check that the low-order bits of the address are appropriate
	358	* for the alignment implied by len.
	359	*/
24f1e32c	360	if (info->address & align)
0067f129 P	361	return -EINVAL;
0067f129 P	362
0067f129 P	363	return 0;
	364	}
	365
9f6b3c2c FW	366	/*
	367	* Dump the debug register contents to the user.
	368	* We can't dump our per cpu values because it
	369	* may contain cpu wide breakpoint, something that
	370	* doesn't belong to the current task.
	371	*
	372	* TODO: include non-ptrace user breakpoints (perf)
	373	*/
	374	void aout_dump_debugregs(struct user *dump)
	375	{
	376	int i;
	377	int dr7 = 0;
	378	struct perf_event *bp;
	379	struct arch_hw_breakpoint *info;
	380	struct thread_struct *thread = &current->thread;
	381
	382	for (i = 0; i < HBP_NUM; i++) {
	383	bp = thread->ptrace_bps[i];
	384
	385	if (bp && !bp->attr.disabled) {
	386	dump->u_debugreg[i] = bp->attr.bp_addr;
	387	info = counter_arch_bp(bp);
	388	dr7 \|= encode_dr7(i, info->len, info->type);
	389	} else {
	390	dump->u_debugreg[i] = 0;
	391	}
	392	}
	393
	394	dump->u_debugreg[4] = 0;
	395	dump->u_debugreg[5] = 0;
	396	dump->u_debugreg[6] = current->thread.debugreg6;
	397
	398	dump->u_debugreg[7] = dr7;
	399	}
68efa37d	400	EXPORT_SYMBOL_GPL(aout_dump_debugregs);
9f6b3c2c	401
24f1e32c FW	402	/*
	403	* Release the user breakpoints used by ptrace
	404	*/
	405	void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
0067f129	406	{
24f1e32c FW	407	int i;
	408	struct thread_struct *t = &tsk->thread;
	409
	410	for (i = 0; i < HBP_NUM; i++) {
	411	unregister_hw_breakpoint(t->ptrace_bps[i]);
	412	t->ptrace_bps[i] = NULL;
	413	}
f7da04c9 ON	414
	415	t->debugreg6 = 0;
	416	t->ptrace_dr7 = 0;
0067f129 P	417	}
0067f129 P	418
24f1e32c	419	void hw_breakpoint_restore(void)
0067f129	420	{
0a3aee0d TH	421	set_debugreg(__this_cpu_read(cpu_debugreg[0]), 0);
	422	set_debugreg(__this_cpu_read(cpu_debugreg[1]), 1);
	423	set_debugreg(__this_cpu_read(cpu_debugreg[2]), 2);
	424	set_debugreg(__this_cpu_read(cpu_debugreg[3]), 3);
24f1e32c	425	set_debugreg(current->thread.debugreg6, 6);
0a3aee0d	426	set_debugreg(__this_cpu_read(cpu_dr7), 7);
0067f129	427	}
24f1e32c	428	EXPORT_SYMBOL_GPL(hw_breakpoint_restore);
0067f129 P	429
	430	/*
	431	* Handle debug exception notifications.
	432	*
	433	* Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
	434	*
	435	* NOTIFY_DONE returned if one of the following conditions is true.
	436	* i) When the causative address is from user-space and the exception
	437	* is a valid one, i.e. not triggered as a result of lazy debug register
	438	* switching
	439	* ii) When there are more bits than trap<n> set in DR6 register (such
	440	* as BD, BS or BT) indicating that more than one debug condition is
	441	* met and requires some more action in do_debug().
	442	*
	443	* NOTIFY_STOP returned for all other cases
	444	*
	445	*/
9c54b616	446	static int hw_breakpoint_handler(struct die_args *args)
0067f129 P	447	{
0067f129 P	448	int i, cpu, rc = NOTIFY_STOP;
24f1e32c	449	struct perf_event *bp;
62edab90 P	450	unsigned long dr7, dr6;
	451	unsigned long *dr6_p;
	452
	453	/* The DR6 value is pointed by args->err */
	454	dr6_p = (unsigned long *)ERR_PTR(args->err);
	455	dr6 = *dr6_p;
0067f129	456
6c0aca28 FW	457	/* If it's a single step, TRAP bits are random */
	458	if (dr6 & DR_STEP)
	459	return NOTIFY_DONE;
	460
0067f129 P	461	/* Do an early return if no trap bits are set in DR6 */
	462	if ((dr6 & DR_TRAP_BITS) == 0)
	463	return NOTIFY_DONE;
	464
0067f129 P	465	get_debugreg(dr7, 7);
	466	/* Disable breakpoints during exception handling */
	467	set_debugreg(0UL, 7);
	468	/*
	469	* Assert that local interrupts are disabled
	470	* Reset the DRn bits in the virtualized register value.
	471	* The ptrace trigger routine will add in whatever is needed.
	472	*/
	473	current->thread.debugreg6 &= ~DR_TRAP_BITS;
	474	cpu = get_cpu();
	475
	476	/* Handle all the breakpoints that were triggered */
	477	for (i = 0; i < HBP_NUM; ++i) {
	478	if (likely(!(dr6 & (DR_TRAP0 << i))))
	479	continue;
24f1e32c	480
0067f129	481	/*
24f1e32c FW	482	* The counter may be concurrently released but that can only
	483	* occur from a call_rcu() path. We can then safely fetch
	484	* the breakpoint, use its callback, touch its counter
	485	* while we are in an rcu_read_lock() path.
0067f129	486	*/
24f1e32c FW	487	rcu_read_lock();
	488
	489	bp = per_cpu(bp_per_reg[i], cpu);
62edab90 P	490	/*
	491	* Reset the 'i'th TRAP bit in dr6 to denote completion of
	492	* exception handling
	493	*/
	494	(*dr6_p) &= ~(DR_TRAP0 << i);
0067f129 P	495	/*
0067f129 P	496	* bp can be NULL due to lazy debug register switching
24f1e32c	497	* or due to concurrent perf counter removing.
0067f129	498	*/
24f1e32c FW	499	if (!bp) {
	500	rcu_read_unlock();
	501	break;
	502	}
	503
b326e956	504	perf_bp_event(bp, args->regs);
0067f129	505
0c4519e8 FW	506	/*
	507	* Set up resume flag to avoid breakpoint recursion when
	508	* returning back to origin.
	509	*/
	510	if (bp->hw.info.type == X86_BREAKPOINT_EXECUTE)
	511	args->regs->flags \|= X86_EFLAGS_RF;
	512
24f1e32c	513	rcu_read_unlock();
0067f129	514	}
e0e53db6 P	515	/*
	516	* Further processing in do_debug() is needed for a) user-space
	517	* breakpoints (to generate signals) and b) when the system has
	518	* taken exception due to multiple causes
	519	*/
	520	if ((current->thread.debugreg6 & DR_TRAP_BITS) \|\|
	521	(dr6 & (~DR_TRAP_BITS)))
0067f129 P	522	rc = NOTIFY_DONE;
	523
	524	set_debugreg(dr7, 7);
eadb8a09	525	put_cpu();
24f1e32c	526
0067f129 P	527	return rc;
	528	}
	529
	530	/*
	531	* Handle debug exception notifications.
	532	*/
9c54b616	533	int hw_breakpoint_exceptions_notify(
0067f129 P	534	struct notifier_block unused, unsigned long val, void data)
	535	{
	536	if (val != DIE_DEBUG)
	537	return NOTIFY_DONE;
	538
	539	return hw_breakpoint_handler(data);
	540	}
24f1e32c FW	541
	542	void hw_breakpoint_pmu_read(struct perf_event *bp)
	543	{
	544	/* TODO */
	545	}