[deliverable/linux.git] / arch / x86 / kernel / cpu / perf_event_intel_lbr.c

#include <linux/perf_event.h>
#include <linux/types.h>

#include <asm/perf_event.h>
#include <asm/msr.h>
#include <asm/insn.h>

#include "perf_event.h"

enum {
	LBR_FORMAT_32		= 0x00,
	LBR_FORMAT_LIP		= 0x01,
	LBR_FORMAT_EIP		= 0x02,
	LBR_FORMAT_EIP_FLAGS	= 0x03,
};

/*
 * Intel LBR_SELECT bits
 * Intel Vol3a, April 2011, Section 16.7 Table 16-10
 *
 * Hardware branch filter (not available on all CPUs)
 */
#define LBR_KERNEL_BIT		0 /* do not capture at ring0 */
#define LBR_USER_BIT		1 /* do not capture at ring > 0 */
#define LBR_JCC_BIT		2 /* do not capture conditional branches */
#define LBR_REL_CALL_BIT	3 /* do not capture relative calls */
#define LBR_IND_CALL_BIT	4 /* do not capture indirect calls */
#define LBR_RETURN_BIT		5 /* do not capture near returns */
#define LBR_IND_JMP_BIT		6 /* do not capture indirect jumps */
#define LBR_REL_JMP_BIT		7 /* do not capture relative jumps */
#define LBR_FAR_BIT		8 /* do not capture far branches */

#define LBR_KERNEL	(1 << LBR_KERNEL_BIT)
#define LBR_USER	(1 << LBR_USER_BIT)
#define LBR_JCC		(1 << LBR_JCC_BIT)
#define LBR_REL_CALL	(1 << LBR_REL_CALL_BIT)
#define LBR_IND_CALL	(1 << LBR_IND_CALL_BIT)
#define LBR_RETURN	(1 << LBR_RETURN_BIT)
#define LBR_REL_JMP	(1 << LBR_REL_JMP_BIT)
#define LBR_IND_JMP	(1 << LBR_IND_JMP_BIT)
#define LBR_FAR		(1 << LBR_FAR_BIT)

#define LBR_PLM (LBR_KERNEL | LBR_USER)

#define LBR_SEL_MASK	0x1ff	/* valid bits in LBR_SELECT */
#define LBR_NOT_SUPP	-1	/* LBR filter not supported */
#define LBR_IGN		0	/* ignored */

#define LBR_ANY		 \
	(LBR_JCC	|\
	 LBR_REL_CALL	|\
	 LBR_IND_CALL	|\
	 LBR_RETURN	|\
	 LBR_REL_JMP	|\
	 LBR_IND_JMP	|\
	 LBR_FAR)

#define LBR_FROM_FLAG_MISPRED  (1ULL << 63)

#define for_each_branch_sample_type(x) \
	for ((x) = PERF_SAMPLE_BRANCH_USER; \
	     (x) < PERF_SAMPLE_BRANCH_MAX; (x) <<= 1)

/*
 * x86control flow change classification
 * x86control flow changes include branches, interrupts, traps, faults
 */
enum {
	X86_BR_NONE     = 0,      /* unknown */

	X86_BR_USER     = 1 << 0, /* branch target is user */
	X86_BR_KERNEL   = 1 << 1, /* branch target is kernel */

	X86_BR_CALL     = 1 << 2, /* call */
	X86_BR_RET      = 1 << 3, /* return */
	X86_BR_SYSCALL  = 1 << 4, /* syscall */
	X86_BR_SYSRET   = 1 << 5, /* syscall return */
	X86_BR_INT      = 1 << 6, /* sw interrupt */
	X86_BR_IRET     = 1 << 7, /* return from interrupt */
	X86_BR_JCC      = 1 << 8, /* conditional */
	X86_BR_JMP      = 1 << 9, /* jump */
	X86_BR_IRQ      = 1 << 10,/* hw interrupt or trap or fault */
	X86_BR_IND_CALL = 1 << 11,/* indirect calls */
};

#define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)

#define X86_BR_ANY       \
	(X86_BR_CALL    |\
	 X86_BR_RET     |\
	 X86_BR_SYSCALL |\
	 X86_BR_SYSRET  |\
	 X86_BR_INT     |\
	 X86_BR_IRET    |\
	 X86_BR_JCC     |\
	 X86_BR_JMP	 |\
	 X86_BR_IRQ	 |\
	 X86_BR_IND_CALL)

#define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)

#define X86_BR_ANY_CALL		 \
	(X86_BR_CALL		|\
	 X86_BR_IND_CALL	|\
	 X86_BR_SYSCALL		|\
	 X86_BR_IRQ		|\
	 X86_BR_INT)

static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);

/*
 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
 * otherwise it becomes near impossible to get a reliable stack.
 */

static void __intel_pmu_lbr_enable(void)
{
	u64 debugctl;
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

	if (cpuc->lbr_sel)
		wrmsrl(MSR_LBR_SELECT, cpuc->lbr_sel->config);

	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
	debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
}

static void __intel_pmu_lbr_disable(void)
{
	u64 debugctl;

	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
	debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
}

static void intel_pmu_lbr_reset_32(void)
{
	int i;

	for (i = 0; i < x86_pmu.lbr_nr; i++)
		wrmsrl(x86_pmu.lbr_from + i, 0);
}

static void intel_pmu_lbr_reset_64(void)
{
	int i;

	for (i = 0; i < x86_pmu.lbr_nr; i++) {
		wrmsrl(x86_pmu.lbr_from + i, 0);
		wrmsrl(x86_pmu.lbr_to   + i, 0);
	}
}

void intel_pmu_lbr_reset(void)
{
	if (!x86_pmu.lbr_nr)
		return;

	if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
		intel_pmu_lbr_reset_32();
	else
		intel_pmu_lbr_reset_64();
}

void intel_pmu_lbr_enable(struct perf_event *event)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

	if (!x86_pmu.lbr_nr)
		return;

	/*
	 * Reset the LBR stack if we changed task context to
	 * avoid data leaks.
	 */
	if (event->ctx->task && cpuc->lbr_context != event->ctx) {
		intel_pmu_lbr_reset();
		cpuc->lbr_context = event->ctx;
	}
	cpuc->br_sel = event->hw.branch_reg.reg;

	cpuc->lbr_users++;
}

void intel_pmu_lbr_disable(struct perf_event *event)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

	if (!x86_pmu.lbr_nr)
		return;

	cpuc->lbr_users--;
	WARN_ON_ONCE(cpuc->lbr_users < 0);

	if (cpuc->enabled && !cpuc->lbr_users) {
		__intel_pmu_lbr_disable();
		/* avoid stale pointer */
		cpuc->lbr_context = NULL;
	}
}

void intel_pmu_lbr_enable_all(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

	if (cpuc->lbr_users)
		__intel_pmu_lbr_enable();
}

void intel_pmu_lbr_disable_all(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

	if (cpuc->lbr_users)
		__intel_pmu_lbr_disable();
}

/*
 * TOS = most recently recorded branch
 */
static inline u64 intel_pmu_lbr_tos(void)
{
	u64 tos;

	rdmsrl(x86_pmu.lbr_tos, tos);

	return tos;
}

static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
{
	unsigned long mask = x86_pmu.lbr_nr - 1;
	u64 tos = intel_pmu_lbr_tos();
	int i;

	for (i = 0; i < x86_pmu.lbr_nr; i++) {
		unsigned long lbr_idx = (tos - i) & mask;
		union {
			struct {
				u32 from;
				u32 to;
			};
			u64     lbr;
		} msr_lastbranch;

		rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);

		cpuc->lbr_entries[i].from	= msr_lastbranch.from;
		cpuc->lbr_entries[i].to		= msr_lastbranch.to;
		cpuc->lbr_entries[i].mispred	= 0;
		cpuc->lbr_entries[i].predicted	= 0;
		cpuc->lbr_entries[i].reserved	= 0;
	}
	cpuc->lbr_stack.nr = i;
}

/*
 * Due to lack of segmentation in Linux the effective address (offset)
 * is the same as the linear address, allowing us to merge the LIP and EIP
 * LBR formats.
 */
static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
{
	unsigned long mask = x86_pmu.lbr_nr - 1;
	int lbr_format = x86_pmu.intel_cap.lbr_format;
	u64 tos = intel_pmu_lbr_tos();
	int i;

	for (i = 0; i < x86_pmu.lbr_nr; i++) {
		unsigned long lbr_idx = (tos - i) & mask;
		u64 from, to, mis = 0, pred = 0;

		rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
		rdmsrl(x86_pmu.lbr_to   + lbr_idx, to);

		if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
			mis = !!(from & LBR_FROM_FLAG_MISPRED);
			pred = !mis;
			from = (u64)((((s64)from) << 1) >> 1);
		}

		cpuc->lbr_entries[i].from	= from;
		cpuc->lbr_entries[i].to		= to;
		cpuc->lbr_entries[i].mispred	= mis;
		cpuc->lbr_entries[i].predicted	= pred;
		cpuc->lbr_entries[i].reserved	= 0;
	}
	cpuc->lbr_stack.nr = i;
}

void intel_pmu_lbr_read(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

	if (!cpuc->lbr_users)
		return;

	if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
		intel_pmu_lbr_read_32(cpuc);
	else
		intel_pmu_lbr_read_64(cpuc);

	intel_pmu_lbr_filter(cpuc);
}

/*
 * SW filter is used:
 * - in case there is no HW filter
 * - in case the HW filter has errata or limitations
 */
static void intel_pmu_setup_sw_lbr_filter(struct perf_event *event)
{
	u64 br_type = event->attr.branch_sample_type;
	int mask = 0;

	if (br_type & PERF_SAMPLE_BRANCH_USER)
		mask |= X86_BR_USER;

	if (br_type & PERF_SAMPLE_BRANCH_KERNEL)
		mask |= X86_BR_KERNEL;

	/* we ignore BRANCH_HV here */

	if (br_type & PERF_SAMPLE_BRANCH_ANY)
		mask |= X86_BR_ANY;

	if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL)
		mask |= X86_BR_ANY_CALL;

	if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
		mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET;

	if (br_type & PERF_SAMPLE_BRANCH_IND_CALL)
		mask |= X86_BR_IND_CALL;
	/*
	 * stash actual user request into reg, it may
	 * be used by fixup code for some CPU
	 */
	event->hw.branch_reg.reg = mask;
}

/*
 * setup the HW LBR filter
 * Used only when available, may not be enough to disambiguate
 * all branches, may need the help of the SW filter
 */
static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event)
{
	struct hw_perf_event_extra *reg;
	u64 br_type = event->attr.branch_sample_type;
	u64 mask = 0, m;
	u64 v;

	for_each_branch_sample_type(m) {
		if (!(br_type & m))
			continue;

		v = x86_pmu.lbr_sel_map[m];
		if (v == LBR_NOT_SUPP)
			return -EOPNOTSUPP;

		if (v != LBR_IGN)
			mask |= v;
	}
	reg = &event->hw.branch_reg;
	reg->idx = EXTRA_REG_LBR;

	/* LBR_SELECT operates in suppress mode so invert mask */
	reg->config = ~mask & x86_pmu.lbr_sel_mask;

	return 0;
}

int intel_pmu_setup_lbr_filter(struct perf_event *event)
{
	int ret = 0;

	/*
	 * no LBR on this PMU
	 */
	if (!x86_pmu.lbr_nr)
		return -EOPNOTSUPP;

	/*
	 * setup SW LBR filter
	 */
	intel_pmu_setup_sw_lbr_filter(event);

	/*
	 * setup HW LBR filter, if any
	 */
	if (x86_pmu.lbr_sel_map)
		ret = intel_pmu_setup_hw_lbr_filter(event);

	return ret;
}

/*
 * return the type of control flow change at address "from"
 * intruction is not necessarily a branch (in case of interrupt).
 *
 * The branch type returned also includes the priv level of the
 * target of the control flow change (X86_BR_USER, X86_BR_KERNEL).
 *
 * If a branch type is unknown OR the instruction cannot be
 * decoded (e.g., text page not present), then X86_BR_NONE is
 * returned.
 */
static int branch_type(unsigned long from, unsigned long to)
{
	struct insn insn;
	void *addr;
	int bytes, size = MAX_INSN_SIZE;
	int ret = X86_BR_NONE;
	int ext, to_plm, from_plm;
	u8 buf[MAX_INSN_SIZE];
	int is64 = 0;

	to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER;
	from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER;

	/*
	 * maybe zero if lbr did not fill up after a reset by the time
	 * we get a PMU interrupt
	 */
	if (from == 0 || to == 0)
		return X86_BR_NONE;

	if (from_plm == X86_BR_USER) {
		/*
		 * can happen if measuring at the user level only
		 * and we interrupt in a kernel thread, e.g., idle.
		 */
		if (!current->mm)
			return X86_BR_NONE;

		/* may fail if text not present */
		bytes = copy_from_user_nmi(buf, (void __user *)from, size);
		if (bytes != size)
			return X86_BR_NONE;

		addr = buf;
	} else
		addr = (void *)from;

	/*
	 * decoder needs to know the ABI especially
	 * on 64-bit systems running 32-bit apps
	 */
#ifdef CONFIG_X86_64
	is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32);
#endif
	insn_init(&insn, addr, is64);
	insn_get_opcode(&insn);

	switch (insn.opcode.bytes[0]) {
	case 0xf:
		switch (insn.opcode.bytes[1]) {
		case 0x05: /* syscall */
		case 0x34: /* sysenter */
			ret = X86_BR_SYSCALL;
			break;
		case 0x07: /* sysret */
		case 0x35: /* sysexit */
			ret = X86_BR_SYSRET;
			break;
		case 0x80 ... 0x8f: /* conditional */
			ret = X86_BR_JCC;
			break;
		default:
			ret = X86_BR_NONE;
		}
		break;
	case 0x70 ... 0x7f: /* conditional */
		ret = X86_BR_JCC;
		break;
	case 0xc2: /* near ret */
	case 0xc3: /* near ret */
	case 0xca: /* far ret */
	case 0xcb: /* far ret */
		ret = X86_BR_RET;
		break;
	case 0xcf: /* iret */
		ret = X86_BR_IRET;
		break;
	case 0xcc ... 0xce: /* int */
		ret = X86_BR_INT;
		break;
	case 0xe8: /* call near rel */
	case 0x9a: /* call far absolute */
		ret = X86_BR_CALL;
		break;
	case 0xe0 ... 0xe3: /* loop jmp */
		ret = X86_BR_JCC;
		break;
	case 0xe9 ... 0xeb: /* jmp */
		ret = X86_BR_JMP;
		break;
	case 0xff: /* call near absolute, call far absolute ind */
		insn_get_modrm(&insn);
		ext = (insn.modrm.bytes[0] >> 3) & 0x7;
		switch (ext) {
		case 2: /* near ind call */
		case 3: /* far ind call */
			ret = X86_BR_IND_CALL;
			break;
		case 4:
		case 5:
			ret = X86_BR_JMP;
			break;
		}
		break;
	default:
		ret = X86_BR_NONE;
	}
	/*
	 * interrupts, traps, faults (and thus ring transition) may
	 * occur on any instructions. Thus, to classify them correctly,
	 * we need to first look at the from and to priv levels. If they
	 * are different and to is in the kernel, then it indicates
	 * a ring transition. If the from instruction is not a ring
	 * transition instr (syscall, systenter, int), then it means
	 * it was a irq, trap or fault.
	 *
	 * we have no way of detecting kernel to kernel faults.
	 */
	if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL
	    && ret != X86_BR_SYSCALL && ret != X86_BR_INT)
		ret = X86_BR_IRQ;

	/*
	 * branch priv level determined by target as
	 * is done by HW when LBR_SELECT is implemented
	 */
	if (ret != X86_BR_NONE)
		ret |= to_plm;

	return ret;
}

/*
 * implement actual branch filter based on user demand.
 * Hardware may not exactly satisfy that request, thus
 * we need to inspect opcodes. Mismatched branches are
 * discarded. Therefore, the number of branches returned
 * in PERF_SAMPLE_BRANCH_STACK sample may vary.
 */
static void
intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
{
	u64 from, to;
	int br_sel = cpuc->br_sel;
	int i, j, type;
	bool compress = false;

	/* if sampling all branches, then nothing to filter */
	if ((br_sel & X86_BR_ALL) == X86_BR_ALL)
		return;

	for (i = 0; i < cpuc->lbr_stack.nr; i++) {

		from = cpuc->lbr_entries[i].from;
		to = cpuc->lbr_entries[i].to;

		type = branch_type(from, to);

		/* if type does not correspond, then discard */
		if (type == X86_BR_NONE || (br_sel & type) != type) {
			cpuc->lbr_entries[i].from = 0;
			compress = true;
		}
	}

	if (!compress)
		return;

	/* remove all entries with from=0 */
	for (i = 0; i < cpuc->lbr_stack.nr; ) {
		if (!cpuc->lbr_entries[i].from) {
			j = i;
			while (++j < cpuc->lbr_stack.nr)
				cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];
			cpuc->lbr_stack.nr--;
			if (!cpuc->lbr_entries[i].from)
				continue;
		}
		i++;
	}
}

/*
 * Map interface branch filters onto LBR filters
 */
static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
	[PERF_SAMPLE_BRANCH_ANY]	= LBR_ANY,
	[PERF_SAMPLE_BRANCH_USER]	= LBR_USER,
	[PERF_SAMPLE_BRANCH_KERNEL]	= LBR_KERNEL,
	[PERF_SAMPLE_BRANCH_HV]		= LBR_IGN,
	[PERF_SAMPLE_BRANCH_ANY_RETURN]	= LBR_RETURN | LBR_REL_JMP
					| LBR_IND_JMP | LBR_FAR,
	/*
	 * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches
	 */
	[PERF_SAMPLE_BRANCH_ANY_CALL] =
	 LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR,
	/*
	 * NHM/WSM erratum: must include IND_JMP to capture IND_CALL
	 */
	[PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL | LBR_IND_JMP,
};

static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
	[PERF_SAMPLE_BRANCH_ANY]	= LBR_ANY,
	[PERF_SAMPLE_BRANCH_USER]	= LBR_USER,
	[PERF_SAMPLE_BRANCH_KERNEL]	= LBR_KERNEL,
	[PERF_SAMPLE_BRANCH_HV]		= LBR_IGN,
	[PERF_SAMPLE_BRANCH_ANY_RETURN]	= LBR_RETURN | LBR_FAR,
	[PERF_SAMPLE_BRANCH_ANY_CALL]	= LBR_REL_CALL | LBR_IND_CALL
					| LBR_FAR,
	[PERF_SAMPLE_BRANCH_IND_CALL]	= LBR_IND_CALL,
};

/* core */
void intel_pmu_lbr_init_core(void)
{
	x86_pmu.lbr_nr     = 4;
	x86_pmu.lbr_tos    = MSR_LBR_TOS;
	x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
	x86_pmu.lbr_to     = MSR_LBR_CORE_TO;

	/*
	 * SW branch filter usage:
	 * - compensate for lack of HW filter
	 */
	pr_cont("4-deep LBR, ");
}

/* nehalem/westmere */
void intel_pmu_lbr_init_nhm(void)
{
	x86_pmu.lbr_nr     = 16;
	x86_pmu.lbr_tos    = MSR_LBR_TOS;
	x86_pmu.lbr_from   = MSR_LBR_NHM_FROM;
	x86_pmu.lbr_to     = MSR_LBR_NHM_TO;

	x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
	x86_pmu.lbr_sel_map  = nhm_lbr_sel_map;

	/*
	 * SW branch filter usage:
	 * - workaround LBR_SEL errata (see above)
	 * - support syscall, sysret capture.
	 *   That requires LBR_FAR but that means far
	 *   jmp need to be filtered out
	 */
	pr_cont("16-deep LBR, ");
}

/* sandy bridge */
void intel_pmu_lbr_init_snb(void)
{
	x86_pmu.lbr_nr	 = 16;
	x86_pmu.lbr_tos	 = MSR_LBR_TOS;
	x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
	x86_pmu.lbr_to   = MSR_LBR_NHM_TO;

	x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
	x86_pmu.lbr_sel_map  = snb_lbr_sel_map;

	/*
	 * SW branch filter usage:
	 * - support syscall, sysret capture.
	 *   That requires LBR_FAR but that means far
	 *   jmp need to be filtered out
	 */
	pr_cont("16-deep LBR, ");
}

/* atom */
void intel_pmu_lbr_init_atom(void)
{
	/*
	 * only models starting at stepping 10 seems
	 * to have an operational LBR which can freeze
	 * on PMU interrupt
	 */
	if (boot_cpu_data.x86_model == 28
	    && boot_cpu_data.x86_mask < 10) {
		pr_cont("LBR disabled due to erratum");
		return;
	}

	x86_pmu.lbr_nr	   = 8;
	x86_pmu.lbr_tos    = MSR_LBR_TOS;
	x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
	x86_pmu.lbr_to     = MSR_LBR_CORE_TO;

	/*
	 * SW branch filter usage:
	 * - compensate for lack of HW filter
	 */
	pr_cont("8-deep LBR, ");
}
Commit	Line	Data
de0428a7 KW	1	#include <linux/perf_event.h>
	2	#include <linux/types.h>
	3
	4	#include <asm/perf_event.h>
	5	#include <asm/msr.h>
3e702ff6	6	#include <asm/insn.h>
de0428a7 KW	7
de0428a7 KW	8	#include "perf_event.h"
caff2bef PZ	9
	10	enum {
	11	LBR_FORMAT_32 = 0x00,
	12	LBR_FORMAT_LIP = 0x01,
	13	LBR_FORMAT_EIP = 0x02,
	14	LBR_FORMAT_EIP_FLAGS = 0x03,
	15	};
	16
c5cc2cd9 SE	17	/*
	18	* Intel LBR_SELECT bits
	19	* Intel Vol3a, April 2011, Section 16.7 Table 16-10
	20	*
	21	* Hardware branch filter (not available on all CPUs)
	22	*/
	23	#define LBR_KERNEL_BIT 0 /* do not capture at ring0 */
	24	#define LBR_USER_BIT 1 /* do not capture at ring > 0 */
	25	#define LBR_JCC_BIT 2 /* do not capture conditional branches */
	26	#define LBR_REL_CALL_BIT 3 /* do not capture relative calls */
	27	#define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */
	28	#define LBR_RETURN_BIT 5 /* do not capture near returns */
	29	#define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */
	30	#define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */
	31	#define LBR_FAR_BIT 8 /* do not capture far branches */
	32
	33	#define LBR_KERNEL (1 << LBR_KERNEL_BIT)
	34	#define LBR_USER (1 << LBR_USER_BIT)
	35	#define LBR_JCC (1 << LBR_JCC_BIT)
	36	#define LBR_REL_CALL (1 << LBR_REL_CALL_BIT)
	37	#define LBR_IND_CALL (1 << LBR_IND_CALL_BIT)
	38	#define LBR_RETURN (1 << LBR_RETURN_BIT)
	39	#define LBR_REL_JMP (1 << LBR_REL_JMP_BIT)
	40	#define LBR_IND_JMP (1 << LBR_IND_JMP_BIT)
	41	#define LBR_FAR (1 << LBR_FAR_BIT)
	42
	43	#define LBR_PLM (LBR_KERNEL \| LBR_USER)
	44
	45	#define LBR_SEL_MASK 0x1ff /* valid bits in LBR_SELECT */
	46	#define LBR_NOT_SUPP -1 /* LBR filter not supported */
	47	#define LBR_IGN 0 /* ignored */
	48
	49	#define LBR_ANY \
	50	(LBR_JCC \|\
	51	LBR_REL_CALL \|\
	52	LBR_IND_CALL \|\
	53	LBR_RETURN \|\
	54	LBR_REL_JMP \|\
	55	LBR_IND_JMP \|\
	56	LBR_FAR)
	57
	58	#define LBR_FROM_FLAG_MISPRED (1ULL << 63)
	59
60ce0fbd SE	60	#define for_each_branch_sample_type(x) \
	61	for ((x) = PERF_SAMPLE_BRANCH_USER; \
	62	(x) < PERF_SAMPLE_BRANCH_MAX; (x) <<= 1)
	63
3e702ff6 SE	64	/*
	65	* x86control flow change classification
	66	* x86control flow changes include branches, interrupts, traps, faults
	67	*/
	68	enum {
	69	X86_BR_NONE = 0, /* unknown */
	70
	71	X86_BR_USER = 1 << 0, /* branch target is user */
	72	X86_BR_KERNEL = 1 << 1, /* branch target is kernel */
	73
	74	X86_BR_CALL = 1 << 2, /* call */
	75	X86_BR_RET = 1 << 3, /* return */
	76	X86_BR_SYSCALL = 1 << 4, /* syscall */
	77	X86_BR_SYSRET = 1 << 5, /* syscall return */
	78	X86_BR_INT = 1 << 6, /* sw interrupt */
	79	X86_BR_IRET = 1 << 7, /* return from interrupt */
	80	X86_BR_JCC = 1 << 8, /* conditional */
	81	X86_BR_JMP = 1 << 9, /* jump */
	82	X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */
	83	X86_BR_IND_CALL = 1 << 11,/* indirect calls */
	84	};
	85
	86	#define X86_BR_PLM (X86_BR_USER \| X86_BR_KERNEL)
	87
	88	#define X86_BR_ANY \
	89	(X86_BR_CALL \|\
	90	X86_BR_RET \|\
	91	X86_BR_SYSCALL \|\
	92	X86_BR_SYSRET \|\
	93	X86_BR_INT \|\
	94	X86_BR_IRET \|\
	95	X86_BR_JCC \|\
	96	X86_BR_JMP \|\
	97	X86_BR_IRQ \|\
	98	X86_BR_IND_CALL)
	99
	100	#define X86_BR_ALL (X86_BR_PLM \| X86_BR_ANY)
	101
	102	#define X86_BR_ANY_CALL \
	103	(X86_BR_CALL \|\
	104	X86_BR_IND_CALL \|\
	105	X86_BR_SYSCALL \|\
	106	X86_BR_IRQ \|\
	107	X86_BR_INT)
	108
	109	static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
	110
caff2bef PZ	111	/*
	112	* We only support LBR implementations that have FREEZE_LBRS_ON_PMI
	113	* otherwise it becomes near impossible to get a reliable stack.
	114	*/
	115
caff2bef PZ	116	static void __intel_pmu_lbr_enable(void)
	117	{
	118	u64 debugctl;
60ce0fbd SE	119	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	120
	121	if (cpuc->lbr_sel)
	122	wrmsrl(MSR_LBR_SELECT, cpuc->lbr_sel->config);
caff2bef PZ	123
caff2bef PZ	124	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
7c5ecaf7	125	debugctl \|= (DEBUGCTLMSR_LBR \| DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
caff2bef PZ	126	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
	127	}
	128
	129	static void __intel_pmu_lbr_disable(void)
	130	{
	131	u64 debugctl;
	132
	133	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
7c5ecaf7	134	debugctl &= ~(DEBUGCTLMSR_LBR \| DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
caff2bef PZ	135	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
	136	}
	137
	138	static void intel_pmu_lbr_reset_32(void)
	139	{
	140	int i;
	141
	142	for (i = 0; i < x86_pmu.lbr_nr; i++)
	143	wrmsrl(x86_pmu.lbr_from + i, 0);
	144	}
	145
	146	static void intel_pmu_lbr_reset_64(void)
	147	{
	148	int i;
	149
	150	for (i = 0; i < x86_pmu.lbr_nr; i++) {
	151	wrmsrl(x86_pmu.lbr_from + i, 0);
	152	wrmsrl(x86_pmu.lbr_to + i, 0);
	153	}
	154	}
	155
de0428a7	156	void intel_pmu_lbr_reset(void)
caff2bef	157	{
74846d35 PZ	158	if (!x86_pmu.lbr_nr)
	159	return;
	160
8db909a7	161	if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
caff2bef PZ	162	intel_pmu_lbr_reset_32();
	163	else
	164	intel_pmu_lbr_reset_64();
	165	}
	166
de0428a7	167	void intel_pmu_lbr_enable(struct perf_event *event)
caff2bef PZ	168	{
	169	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	170
	171	if (!x86_pmu.lbr_nr)
	172	return;
	173
caff2bef	174	/*
b83a46e7 PZ	175	* Reset the LBR stack if we changed task context to
b83a46e7 PZ	176	* avoid data leaks.
caff2bef	177	*/
b83a46e7	178	if (event->ctx->task && cpuc->lbr_context != event->ctx) {
caff2bef PZ	179	intel_pmu_lbr_reset();
	180	cpuc->lbr_context = event->ctx;
	181	}
3e702ff6	182	cpuc->br_sel = event->hw.branch_reg.reg;
caff2bef PZ	183
	184	cpuc->lbr_users++;
	185	}
	186
de0428a7	187	void intel_pmu_lbr_disable(struct perf_event *event)
caff2bef PZ	188	{
	189	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	190
	191	if (!x86_pmu.lbr_nr)
	192	return;
	193
	194	cpuc->lbr_users--;
b83a46e7	195	WARN_ON_ONCE(cpuc->lbr_users < 0);
2df202bf	196
60ce0fbd	197	if (cpuc->enabled && !cpuc->lbr_users) {
2df202bf	198	__intel_pmu_lbr_disable();
60ce0fbd SE	199	/* avoid stale pointer */
	200	cpuc->lbr_context = NULL;
	201	}
caff2bef PZ	202	}
caff2bef PZ	203
de0428a7	204	void intel_pmu_lbr_enable_all(void)
caff2bef PZ	205	{
	206	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	207
	208	if (cpuc->lbr_users)
	209	__intel_pmu_lbr_enable();
	210	}
	211
de0428a7	212	void intel_pmu_lbr_disable_all(void)
caff2bef PZ	213	{
	214	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	215
	216	if (cpuc->lbr_users)
	217	__intel_pmu_lbr_disable();
	218	}
	219
60ce0fbd SE	220	/*
	221	* TOS = most recently recorded branch
	222	*/
caff2bef PZ	223	static inline u64 intel_pmu_lbr_tos(void)
	224	{
	225	u64 tos;
	226
	227	rdmsrl(x86_pmu.lbr_tos, tos);
	228
	229	return tos;
	230	}
	231
	232	static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
	233	{
	234	unsigned long mask = x86_pmu.lbr_nr - 1;
	235	u64 tos = intel_pmu_lbr_tos();
	236	int i;
	237
63fb3f9b	238	for (i = 0; i < x86_pmu.lbr_nr; i++) {
caff2bef PZ	239	unsigned long lbr_idx = (tos - i) & mask;
	240	union {
	241	struct {
	242	u32 from;
	243	u32 to;
	244	};
	245	u64 lbr;
	246	} msr_lastbranch;
	247
	248	rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
	249
bce38cd5 SE	250	cpuc->lbr_entries[i].from = msr_lastbranch.from;
	251	cpuc->lbr_entries[i].to = msr_lastbranch.to;
	252	cpuc->lbr_entries[i].mispred = 0;
	253	cpuc->lbr_entries[i].predicted = 0;
	254	cpuc->lbr_entries[i].reserved = 0;
caff2bef PZ	255	}
	256	cpuc->lbr_stack.nr = i;
	257	}
	258
caff2bef PZ	259	/*
	260	* Due to lack of segmentation in Linux the effective address (offset)
	261	* is the same as the linear address, allowing us to merge the LIP and EIP
	262	* LBR formats.
	263	*/
	264	static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
	265	{
	266	unsigned long mask = x86_pmu.lbr_nr - 1;
8db909a7	267	int lbr_format = x86_pmu.intel_cap.lbr_format;
caff2bef PZ	268	u64 tos = intel_pmu_lbr_tos();
	269	int i;
	270
63fb3f9b	271	for (i = 0; i < x86_pmu.lbr_nr; i++) {
caff2bef	272	unsigned long lbr_idx = (tos - i) & mask;
bce38cd5	273	u64 from, to, mis = 0, pred = 0;
caff2bef PZ	274
	275	rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
	276	rdmsrl(x86_pmu.lbr_to + lbr_idx, to);
	277
8db909a7	278	if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
bce38cd5 SE	279	mis = !!(from & LBR_FROM_FLAG_MISPRED);
bce38cd5 SE	280	pred = !mis;
caff2bef PZ	281	from = (u64)((((s64)from) << 1) >> 1);
	282	}
	283
bce38cd5 SE	284	cpuc->lbr_entries[i].from = from;
	285	cpuc->lbr_entries[i].to = to;
	286	cpuc->lbr_entries[i].mispred = mis;
	287	cpuc->lbr_entries[i].predicted = pred;
	288	cpuc->lbr_entries[i].reserved = 0;
caff2bef PZ	289	}
	290	cpuc->lbr_stack.nr = i;
	291	}
	292
de0428a7	293	void intel_pmu_lbr_read(void)
caff2bef PZ	294	{
	295	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	296
	297	if (!cpuc->lbr_users)
	298	return;
	299
8db909a7	300	if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
caff2bef PZ	301	intel_pmu_lbr_read_32(cpuc);
	302	else
	303	intel_pmu_lbr_read_64(cpuc);
3e702ff6 SE	304
	305	intel_pmu_lbr_filter(cpuc);
	306	}
	307
	308	/*
	309	* SW filter is used:
	310	* - in case there is no HW filter
	311	* - in case the HW filter has errata or limitations
	312	*/
	313	static void intel_pmu_setup_sw_lbr_filter(struct perf_event *event)
	314	{
	315	u64 br_type = event->attr.branch_sample_type;
	316	int mask = 0;
	317
	318	if (br_type & PERF_SAMPLE_BRANCH_USER)
	319	mask \|= X86_BR_USER;
	320
	321	if (br_type & PERF_SAMPLE_BRANCH_KERNEL)
	322	mask \|= X86_BR_KERNEL;
	323
	324	/* we ignore BRANCH_HV here */
	325
	326	if (br_type & PERF_SAMPLE_BRANCH_ANY)
	327	mask \|= X86_BR_ANY;
	328
	329	if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL)
	330	mask \|= X86_BR_ANY_CALL;
	331
	332	if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
	333	mask \|= X86_BR_RET \| X86_BR_IRET \| X86_BR_SYSRET;
	334
	335	if (br_type & PERF_SAMPLE_BRANCH_IND_CALL)
	336	mask \|= X86_BR_IND_CALL;
	337	/*
	338	* stash actual user request into reg, it may
	339	* be used by fixup code for some CPU
	340	*/
	341	event->hw.branch_reg.reg = mask;
caff2bef PZ	342	}
caff2bef PZ	343
60ce0fbd SE	344	/*
	345	* setup the HW LBR filter
	346	* Used only when available, may not be enough to disambiguate
	347	* all branches, may need the help of the SW filter
	348	*/
	349	static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event)
	350	{
	351	struct hw_perf_event_extra *reg;
	352	u64 br_type = event->attr.branch_sample_type;
	353	u64 mask = 0, m;
	354	u64 v;
	355
	356	for_each_branch_sample_type(m) {
	357	if (!(br_type & m))
	358	continue;
	359
	360	v = x86_pmu.lbr_sel_map[m];
	361	if (v == LBR_NOT_SUPP)
	362	return -EOPNOTSUPP;
60ce0fbd	363
3e702ff6 SE	364	if (v != LBR_IGN)
3e702ff6 SE	365	mask \|= v;
60ce0fbd SE	366	}
	367	reg = &event->hw.branch_reg;
	368	reg->idx = EXTRA_REG_LBR;
	369
	370	/* LBR_SELECT operates in suppress mode so invert mask */
	371	reg->config = ~mask & x86_pmu.lbr_sel_mask;
	372
	373	return 0;
	374	}
	375
60ce0fbd SE	376	int intel_pmu_setup_lbr_filter(struct perf_event *event)
60ce0fbd SE	377	{
3e702ff6	378	int ret = 0;
60ce0fbd SE	379
	380	/*
	381	* no LBR on this PMU
	382	*/
	383	if (!x86_pmu.lbr_nr)
	384	return -EOPNOTSUPP;
	385
	386	/*
3e702ff6	387	* setup SW LBR filter
60ce0fbd	388	*/
3e702ff6 SE	389	intel_pmu_setup_sw_lbr_filter(event);
	390
	391	/*
	392	* setup HW LBR filter, if any
	393	*/
	394	if (x86_pmu.lbr_sel_map)
	395	ret = intel_pmu_setup_hw_lbr_filter(event);
	396
	397	return ret;
	398	}
	399
	400	/*
	401	* return the type of control flow change at address "from"
	402	* intruction is not necessarily a branch (in case of interrupt).
	403	*
	404	* The branch type returned also includes the priv level of the
	405	* target of the control flow change (X86_BR_USER, X86_BR_KERNEL).
	406	*
	407	* If a branch type is unknown OR the instruction cannot be
	408	* decoded (e.g., text page not present), then X86_BR_NONE is
	409	* returned.
	410	*/
	411	static int branch_type(unsigned long from, unsigned long to)
	412	{
	413	struct insn insn;
	414	void *addr;
	415	int bytes, size = MAX_INSN_SIZE;
	416	int ret = X86_BR_NONE;
	417	int ext, to_plm, from_plm;
	418	u8 buf[MAX_INSN_SIZE];
	419	int is64 = 0;
	420
	421	to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER;
	422	from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER;
	423
	424	/*
	425	* maybe zero if lbr did not fill up after a reset by the time
	426	* we get a PMU interrupt
	427	*/
	428	if (from == 0 \|\| to == 0)
	429	return X86_BR_NONE;
	430
	431	if (from_plm == X86_BR_USER) {
	432	/*
	433	* can happen if measuring at the user level only
	434	* and we interrupt in a kernel thread, e.g., idle.
	435	*/
	436	if (!current->mm)
	437	return X86_BR_NONE;
	438
	439	/* may fail if text not present */
	440	bytes = copy_from_user_nmi(buf, (void __user *)from, size);
	441	if (bytes != size)
	442	return X86_BR_NONE;
	443
	444	addr = buf;
	445	} else
	446	addr = (void *)from;
	447
	448	/*
	449	* decoder needs to know the ABI especially
	450	* on 64-bit systems running 32-bit apps
	451	*/
	452	#ifdef CONFIG_X86_64
453	is64 = kernel_ip((unsigned long)addr) \|\| !test_thread_flag(TIF_IA32);
454	#endif
455	insn_init(&insn, addr, is64);
456	insn_get_opcode(&insn);
457
458	switch (insn.opcode.bytes[0]) {
459	case 0xf:
460	switch (insn.opcode.bytes[1]) {
461	case 0x05: /* syscall */
462	case 0x34: /* sysenter */
463	ret = X86_BR_SYSCALL;
464	break;
465	case 0x07: /* sysret */
466	case 0x35: /* sysexit */
467	ret = X86_BR_SYSRET;
468	break;
469	case 0x80 ... 0x8f: /* conditional */
470	ret = X86_BR_JCC;
471	break;
472	default:
473	ret = X86_BR_NONE;
474	}
475	break;
476	case 0x70 ... 0x7f: /* conditional */
477	ret = X86_BR_JCC;
478	break;
479	case 0xc2: /* near ret */
480	case 0xc3: /* near ret */
481	case 0xca: /* far ret */
482	case 0xcb: /* far ret */
483	ret = X86_BR_RET;
484	break;
485	case 0xcf: /* iret */
486	ret = X86_BR_IRET;
487	break;
488	case 0xcc ... 0xce: /* int */
489	ret = X86_BR_INT;
490	break;
491	case 0xe8: /* call near rel */
492	case 0x9a: /* call far absolute */
493	ret = X86_BR_CALL;
494	break;
495	case 0xe0 ... 0xe3: /* loop jmp */
496	ret = X86_BR_JCC;
497	break;
498	case 0xe9 ... 0xeb: /* jmp */
499	ret = X86_BR_JMP;
500	break;
501	case 0xff: /* call near absolute, call far absolute ind */
502	insn_get_modrm(&insn);
503	ext = (insn.modrm.bytes[0] >> 3) & 0x7;
504	switch (ext) {
505	case 2: /* near ind call */
506	case 3: /* far ind call */
507	ret = X86_BR_IND_CALL;
508	break;
509	case 4:
510	case 5:
511	ret = X86_BR_JMP;
512	break;
513	}
514	break;
515	default:
516	ret = X86_BR_NONE;
60ce0fbd SE	517	}
60ce0fbd SE	518	/*
3e702ff6 SE	519	* interrupts, traps, faults (and thus ring transition) may
	520	* occur on any instructions. Thus, to classify them correctly,
	521	* we need to first look at the from and to priv levels. If they
	522	* are different and to is in the kernel, then it indicates
	523	* a ring transition. If the from instruction is not a ring
	524	* transition instr (syscall, systenter, int), then it means
	525	* it was a irq, trap or fault.
	526	*
	527	* we have no way of detecting kernel to kernel faults.
	528	*/
	529	if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL
	530	&& ret != X86_BR_SYSCALL && ret != X86_BR_INT)
	531	ret = X86_BR_IRQ;
	532
	533	/*
	534	* branch priv level determined by target as
	535	* is done by HW when LBR_SELECT is implemented
60ce0fbd	536	*/
3e702ff6 SE	537	if (ret != X86_BR_NONE)
3e702ff6 SE	538	ret \|= to_plm;
60ce0fbd	539
3e702ff6 SE	540	return ret;
	541	}
	542
	543	/*
	544	* implement actual branch filter based on user demand.
	545	* Hardware may not exactly satisfy that request, thus
	546	* we need to inspect opcodes. Mismatched branches are
	547	* discarded. Therefore, the number of branches returned
	548	* in PERF_SAMPLE_BRANCH_STACK sample may vary.
	549	*/
	550	static void
	551	intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
	552	{
	553	u64 from, to;
	554	int br_sel = cpuc->br_sel;
	555	int i, j, type;
	556	bool compress = false;
	557
	558	/* if sampling all branches, then nothing to filter */
	559	if ((br_sel & X86_BR_ALL) == X86_BR_ALL)
	560	return;
	561
	562	for (i = 0; i < cpuc->lbr_stack.nr; i++) {
	563
	564	from = cpuc->lbr_entries[i].from;
	565	to = cpuc->lbr_entries[i].to;
	566
	567	type = branch_type(from, to);
	568
	569	/* if type does not correspond, then discard */
	570	if (type == X86_BR_NONE \|\| (br_sel & type) != type) {
	571	cpuc->lbr_entries[i].from = 0;
	572	compress = true;
	573	}
	574	}
	575
	576	if (!compress)
	577	return;
	578
	579	/* remove all entries with from=0 */
	580	for (i = 0; i < cpuc->lbr_stack.nr; ) {
	581	if (!cpuc->lbr_entries[i].from) {
	582	j = i;
	583	while (++j < cpuc->lbr_stack.nr)
	584	cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];
	585	cpuc->lbr_stack.nr--;
	586	if (!cpuc->lbr_entries[i].from)
	587	continue;
	588	}
	589	i++;
	590	}
60ce0fbd SE	591	}
60ce0fbd SE	592
c5cc2cd9 SE	593	/*
	594	* Map interface branch filters onto LBR filters
	595	*/
	596	static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
	597	[PERF_SAMPLE_BRANCH_ANY] = LBR_ANY,
	598	[PERF_SAMPLE_BRANCH_USER] = LBR_USER,
	599	[PERF_SAMPLE_BRANCH_KERNEL] = LBR_KERNEL,
	600	[PERF_SAMPLE_BRANCH_HV] = LBR_IGN,
	601	[PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN \| LBR_REL_JMP
	602	\| LBR_IND_JMP \| LBR_FAR,
	603	/*
	604	* NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches
	605	*/
	606	[PERF_SAMPLE_BRANCH_ANY_CALL] =
	607	LBR_REL_CALL \| LBR_IND_CALL \| LBR_REL_JMP \| LBR_IND_JMP \| LBR_FAR,
	608	/*
	609	* NHM/WSM erratum: must include IND_JMP to capture IND_CALL
	610	*/
	611	[PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL \| LBR_IND_JMP,
	612	};
	613
	614	static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
	615	[PERF_SAMPLE_BRANCH_ANY] = LBR_ANY,
	616	[PERF_SAMPLE_BRANCH_USER] = LBR_USER,
	617	[PERF_SAMPLE_BRANCH_KERNEL] = LBR_KERNEL,
	618	[PERF_SAMPLE_BRANCH_HV] = LBR_IGN,
	619	[PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN \| LBR_FAR,
	620	[PERF_SAMPLE_BRANCH_ANY_CALL] = LBR_REL_CALL \| LBR_IND_CALL
	621	\| LBR_FAR,
	622	[PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL,
	623	};
	624
	625	/* core */
de0428a7	626	void intel_pmu_lbr_init_core(void)
caff2bef	627	{
caff2bef	628	x86_pmu.lbr_nr = 4;
225ce539 SE	629	x86_pmu.lbr_tos = MSR_LBR_TOS;
	630	x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
	631	x86_pmu.lbr_to = MSR_LBR_CORE_TO;
c5cc2cd9	632
3e702ff6 SE	633	/*
	634	* SW branch filter usage:
	635	* - compensate for lack of HW filter
	636	*/
c5cc2cd9	637	pr_cont("4-deep LBR, ");
caff2bef PZ	638	}
caff2bef PZ	639
c5cc2cd9	640	/* nehalem/westmere */
de0428a7	641	void intel_pmu_lbr_init_nhm(void)
caff2bef	642	{
caff2bef	643	x86_pmu.lbr_nr = 16;
225ce539 SE	644	x86_pmu.lbr_tos = MSR_LBR_TOS;
	645	x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
	646	x86_pmu.lbr_to = MSR_LBR_NHM_TO;
c5cc2cd9 SE	647
	648	x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
	649	x86_pmu.lbr_sel_map = nhm_lbr_sel_map;
	650
3e702ff6 SE	651	/*
	652	* SW branch filter usage:
	653	* - workaround LBR_SEL errata (see above)
	654	* - support syscall, sysret capture.
	655	* That requires LBR_FAR but that means far
	656	* jmp need to be filtered out
	657	*/
c5cc2cd9	658	pr_cont("16-deep LBR, ");
caff2bef PZ	659	}
caff2bef PZ	660
c5cc2cd9 SE	661	/* sandy bridge */
	662	void intel_pmu_lbr_init_snb(void)
	663	{
	664	x86_pmu.lbr_nr = 16;
	665	x86_pmu.lbr_tos = MSR_LBR_TOS;
	666	x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
	667	x86_pmu.lbr_to = MSR_LBR_NHM_TO;
	668
	669	x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
	670	x86_pmu.lbr_sel_map = snb_lbr_sel_map;
	671
3e702ff6 SE	672	/*
	673	* SW branch filter usage:
	674	* - support syscall, sysret capture.
	675	* That requires LBR_FAR but that means far
	676	* jmp need to be filtered out
	677	*/
c5cc2cd9 SE	678	pr_cont("16-deep LBR, ");
	679	}
	680
	681	/* atom */
de0428a7	682	void intel_pmu_lbr_init_atom(void)
caff2bef	683	{
88c9a65e SE	684	/*
	685	* only models starting at stepping 10 seems
	686	* to have an operational LBR which can freeze
	687	* on PMU interrupt
	688	*/
3ec18cd8 SE	689	if (boot_cpu_data.x86_model == 28
3ec18cd8 SE	690	&& boot_cpu_data.x86_mask < 10) {
88c9a65e SE	691	pr_cont("LBR disabled due to erratum");
	692	return;
	693	}
	694
caff2bef	695	x86_pmu.lbr_nr = 8;
225ce539 SE	696	x86_pmu.lbr_tos = MSR_LBR_TOS;
	697	x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
	698	x86_pmu.lbr_to = MSR_LBR_CORE_TO;
c5cc2cd9	699
3e702ff6 SE	700	/*
	701	* SW branch filter usage:
	702	* - compensate for lack of HW filter
	703	*/
c5cc2cd9	704	pr_cont("8-deep LBR, ");
caff2bef	705	}