[deliverable/linux.git] / arch / x86 / oprofile / op_model_amd.c

/*
 * @file op_model_amd.c
 * athlon / K7 / K8 / Family 10h model-specific MSR operations
 *
 * @remark Copyright 2002-2009 OProfile authors
 * @remark Read the file COPYING
 *
 * @author John Levon
 * @author Philippe Elie
 * @author Graydon Hoare
 * @author Robert Richter <robert.richter@amd.com>
 * @author Barry Kasindorf <barry.kasindorf@amd.com>
 * @author Jason Yeh <jason.yeh@amd.com>
 * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
 */

#include <linux/oprofile.h>
#include <linux/device.h>
#include <linux/pci.h>
#include <linux/percpu.h>

#include <asm/ptrace.h>
#include <asm/msr.h>
#include <asm/nmi.h>
#include <asm/apic.h>
#include <asm/processor.h>

#include "op_x86_model.h"
#include "op_counter.h"

#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
#define NUM_VIRT_COUNTERS	32
#else
#define NUM_VIRT_COUNTERS	0
#endif

#define OP_EVENT_MASK			0x0FFF
#define OP_CTR_OVERFLOW			(1ULL<<31)

#define MSR_AMD_EVENTSEL_RESERVED	((0xFFFFFCF0ULL<<32)|(1ULL<<21))

static int num_counters;
static unsigned long reset_value[OP_MAX_COUNTER];

#define IBS_FETCH_SIZE			6
#define IBS_OP_SIZE			12

static u32 ibs_caps;

struct ibs_config {
	unsigned long op_enabled;
	unsigned long fetch_enabled;
	unsigned long max_cnt_fetch;
	unsigned long max_cnt_op;
	unsigned long rand_en;
	unsigned long dispatched_ops;
	unsigned long branch_target;
};

struct ibs_state {
	u64		ibs_op_ctl;
	int		branch_target;
	unsigned long	sample_size;
};

static struct ibs_config ibs_config;
static struct ibs_state ibs_state;

/*
 * IBS randomization macros
 */
#define IBS_RANDOM_BITS			12
#define IBS_RANDOM_MASK			((1ULL << IBS_RANDOM_BITS) - 1)
#define IBS_RANDOM_MAXCNT_OFFSET	(1ULL << (IBS_RANDOM_BITS - 5))

/*
 * 16-bit Linear Feedback Shift Register (LFSR)
 *
 *                       16   14   13    11
 * Feedback polynomial = X  + X  + X  +  X  + 1
 */
static unsigned int lfsr_random(void)
{
	static unsigned int lfsr_value = 0xF00D;
	unsigned int bit;

	/* Compute next bit to shift in */
	bit = ((lfsr_value >> 0) ^
	       (lfsr_value >> 2) ^
	       (lfsr_value >> 3) ^
	       (lfsr_value >> 5)) & 0x0001;

	/* Advance to next register value */
	lfsr_value = (lfsr_value >> 1) | (bit << 15);

	return lfsr_value;
}

/*
 * IBS software randomization
 *
 * The IBS periodic op counter is randomized in software. The lower 12
 * bits of the 20 bit counter are randomized. IbsOpCurCnt is
 * initialized with a 12 bit random value.
 */
static inline u64 op_amd_randomize_ibs_op(u64 val)
{
	unsigned int random = lfsr_random();

	if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
		/*
		 * Work around if the hw can not write to IbsOpCurCnt
		 *
		 * Randomize the lower 8 bits of the 16 bit
		 * IbsOpMaxCnt [15:0] value in the range of -128 to
		 * +127 by adding/subtracting an offset to the
		 * maximum count (IbsOpMaxCnt).
		 *
		 * To avoid over or underflows and protect upper bits
		 * starting at bit 16, the initial value for
		 * IbsOpMaxCnt must fit in the range from 0x0081 to
		 * 0xff80.
		 */
		val += (s8)(random >> 4);
	else
		val |= (u64)(random & IBS_RANDOM_MASK) << 32;

	return val;
}

static inline void
op_amd_handle_ibs(struct pt_regs * const regs,
		  struct op_msrs const * const msrs)
{
	u64 val, ctl;
	struct op_entry entry;

	if (!ibs_caps)
		return;

	if (ibs_config.fetch_enabled) {
		rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
		if (ctl & IBS_FETCH_VAL) {
			rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
			oprofile_write_reserve(&entry, regs, val,
					       IBS_FETCH_CODE, IBS_FETCH_SIZE);
			oprofile_add_data64(&entry, val);
			oprofile_add_data64(&entry, ctl);
			rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
			oprofile_add_data64(&entry, val);
			oprofile_write_commit(&entry);

			/* reenable the IRQ */
			ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT);
			ctl |= IBS_FETCH_ENABLE;
			wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
		}
	}

	if (ibs_config.op_enabled) {
		rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
		if (ctl & IBS_OP_VAL) {
			rdmsrl(MSR_AMD64_IBSOPRIP, val);
			oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE,
					       ibs_state.sample_size);
			oprofile_add_data64(&entry, val);
			rdmsrl(MSR_AMD64_IBSOPDATA, val);
			oprofile_add_data64(&entry, val);
			rdmsrl(MSR_AMD64_IBSOPDATA2, val);
			oprofile_add_data64(&entry, val);
			rdmsrl(MSR_AMD64_IBSOPDATA3, val);
			oprofile_add_data64(&entry, val);
			rdmsrl(MSR_AMD64_IBSDCLINAD, val);
			oprofile_add_data64(&entry, val);
			rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
			oprofile_add_data64(&entry, val);
			if (ibs_state.branch_target) {
				rdmsrl(MSR_AMD64_IBSBRTARGET, val);
				oprofile_add_data(&entry, (unsigned long)val);
			}
			oprofile_write_commit(&entry);

			/* reenable the IRQ */
			ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
			wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
		}
	}
}

static inline void op_amd_start_ibs(void)
{
	u64 val;

	if (!ibs_caps)
		return;

	memset(&ibs_state, 0, sizeof(ibs_state));

	/*
	 * Note: Since the max count settings may out of range we
	 * write back the actual used values so that userland can read
	 * it.
	 */

	if (ibs_config.fetch_enabled) {
		val = ibs_config.max_cnt_fetch >> 4;
		val = min(val, IBS_FETCH_MAX_CNT);
		ibs_config.max_cnt_fetch = val << 4;
		val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
		val |= IBS_FETCH_ENABLE;
		wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
	}

	if (ibs_config.op_enabled) {
		val = ibs_config.max_cnt_op >> 4;
		if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
			/*
			 * IbsOpCurCnt not supported.  See
			 * op_amd_randomize_ibs_op() for details.
			 */
			val = clamp(val, 0x0081ULL, 0xFF80ULL);
			ibs_config.max_cnt_op = val << 4;
		} else {
			/*
			 * The start value is randomized with a
			 * positive offset, we need to compensate it
			 * with the half of the randomized range. Also
			 * avoid underflows.
			 */
			val += IBS_RANDOM_MAXCNT_OFFSET;
			if (ibs_caps & IBS_CAPS_OPCNTEXT)
				val = min(val, IBS_OP_MAX_CNT_EXT);
			else
				val = min(val, IBS_OP_MAX_CNT);
			ibs_config.max_cnt_op =
				(val - IBS_RANDOM_MAXCNT_OFFSET) << 4;
		}
		val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT);
		val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
		val |= IBS_OP_ENABLE;
		ibs_state.ibs_op_ctl = val;
		ibs_state.sample_size = IBS_OP_SIZE;
		if (ibs_config.branch_target) {
			ibs_state.branch_target = 1;
			ibs_state.sample_size++;
		}
		val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
		wrmsrl(MSR_AMD64_IBSOPCTL, val);
	}
}

static void op_amd_stop_ibs(void)
{
	if (!ibs_caps)
		return;

	if (ibs_config.fetch_enabled)
		/* clear max count and enable */
		wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);

	if (ibs_config.op_enabled)
		/* clear max count and enable */
		wrmsrl(MSR_AMD64_IBSOPCTL, 0);
}

#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX

static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
			       struct op_msrs const * const msrs)
{
	u64 val;
	int i;

	/* enable active counters */
	for (i = 0; i < num_counters; ++i) {
		int virt = op_x86_phys_to_virt(i);
		if (!reset_value[virt])
			continue;
		rdmsrl(msrs->controls[i].addr, val);
		val &= model->reserved;
		val |= op_x86_get_ctrl(model, &counter_config[virt]);
		wrmsrl(msrs->controls[i].addr, val);
	}
}

#endif

/* functions for op_amd_spec */

static void op_amd_shutdown(struct op_msrs const * const msrs)
{
	int i;

	for (i = 0; i < num_counters; ++i) {
		if (!msrs->counters[i].addr)
			continue;
		release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
		release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
	}
}

static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
{
	int i;

	for (i = 0; i < num_counters; i++) {
		if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
			goto fail;
		if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
			release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
			goto fail;
		}
		/* both registers must be reserved */
		if (num_counters == AMD64_NUM_COUNTERS_CORE) {
			msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1);
			msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1);
		} else {
			msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
			msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
		}
		continue;
	fail:
		if (!counter_config[i].enabled)
			continue;
		op_x86_warn_reserved(i);
		op_amd_shutdown(msrs);
		return -EBUSY;
	}

	return 0;
}

static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
			      struct op_msrs const * const msrs)
{
	u64 val;
	int i;

	/* setup reset_value */
	for (i = 0; i < OP_MAX_COUNTER; ++i) {
		if (counter_config[i].enabled
		    && msrs->counters[op_x86_virt_to_phys(i)].addr)
			reset_value[i] = counter_config[i].count;
		else
			reset_value[i] = 0;
	}

	/* clear all counters */
	for (i = 0; i < num_counters; ++i) {
		if (!msrs->controls[i].addr)
			continue;
		rdmsrl(msrs->controls[i].addr, val);
		if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
			op_x86_warn_in_use(i);
		val &= model->reserved;
		wrmsrl(msrs->controls[i].addr, val);
		/*
		 * avoid a false detection of ctr overflows in NMI
		 * handler
		 */
		wrmsrl(msrs->counters[i].addr, -1LL);
	}

	/* enable active counters */
	for (i = 0; i < num_counters; ++i) {
		int virt = op_x86_phys_to_virt(i);
		if (!reset_value[virt])
			continue;

		/* setup counter registers */
		wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);

		/* setup control registers */
		rdmsrl(msrs->controls[i].addr, val);
		val &= model->reserved;
		val |= op_x86_get_ctrl(model, &counter_config[virt]);
		wrmsrl(msrs->controls[i].addr, val);
	}
}

static int op_amd_check_ctrs(struct pt_regs * const regs,
			     struct op_msrs const * const msrs)
{
	u64 val;
	int i;

	for (i = 0; i < num_counters; ++i) {
		int virt = op_x86_phys_to_virt(i);
		if (!reset_value[virt])
			continue;
		rdmsrl(msrs->counters[i].addr, val);
		/* bit is clear if overflowed: */
		if (val & OP_CTR_OVERFLOW)
			continue;
		oprofile_add_sample(regs, virt);
		wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
	}

	op_amd_handle_ibs(regs, msrs);

	/* See op_model_ppro.c */
	return 1;
}

static void op_amd_start(struct op_msrs const * const msrs)
{
	u64 val;
	int i;

	for (i = 0; i < num_counters; ++i) {
		if (!reset_value[op_x86_phys_to_virt(i)])
			continue;
		rdmsrl(msrs->controls[i].addr, val);
		val |= ARCH_PERFMON_EVENTSEL_ENABLE;
		wrmsrl(msrs->controls[i].addr, val);
	}

	op_amd_start_ibs();
}

static void op_amd_stop(struct op_msrs const * const msrs)
{
	u64 val;
	int i;

	/*
	 * Subtle: stop on all counters to avoid race with setting our
	 * pm callback
	 */
	for (i = 0; i < num_counters; ++i) {
		if (!reset_value[op_x86_phys_to_virt(i)])
			continue;
		rdmsrl(msrs->controls[i].addr, val);
		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
		wrmsrl(msrs->controls[i].addr, val);
	}

	op_amd_stop_ibs();
}

/*
 * check and reserve APIC extended interrupt LVT offset for IBS if
 * available
 */

static void init_ibs(void)
{
	ibs_caps = get_ibs_caps();

	if (!ibs_caps)
		return;

	printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps);
}

static int (*create_arch_files)(struct dentry *root);

static int setup_ibs_files(struct dentry *root)
{
	struct dentry *dir;
	int ret = 0;

	/* architecture specific files */
	if (create_arch_files)
		ret = create_arch_files(root);

	if (ret)
		return ret;

	if (!ibs_caps)
		return ret;

	/* model specific files */

	/* setup some reasonable defaults */
	memset(&ibs_config, 0, sizeof(ibs_config));
	ibs_config.max_cnt_fetch = 250000;
	ibs_config.max_cnt_op = 250000;

	if (ibs_caps & IBS_CAPS_FETCHSAM) {
		dir = oprofilefs_mkdir(root, "ibs_fetch");
		oprofilefs_create_ulong(dir, "enable",
					&ibs_config.fetch_enabled);
		oprofilefs_create_ulong(dir, "max_count",
					&ibs_config.max_cnt_fetch);
		oprofilefs_create_ulong(dir, "rand_enable",
					&ibs_config.rand_en);
	}

	if (ibs_caps & IBS_CAPS_OPSAM) {
		dir = oprofilefs_mkdir(root, "ibs_op");
		oprofilefs_create_ulong(dir, "enable",
					&ibs_config.op_enabled);
		oprofilefs_create_ulong(dir, "max_count",
					&ibs_config.max_cnt_op);
		if (ibs_caps & IBS_CAPS_OPCNT)
			oprofilefs_create_ulong(dir, "dispatched_ops",
						&ibs_config.dispatched_ops);
		if (ibs_caps & IBS_CAPS_BRNTRGT)
			oprofilefs_create_ulong(dir, "branch_target",
						&ibs_config.branch_target);
	}

	return 0;
}

struct op_x86_model_spec op_amd_spec;

static int op_amd_init(struct oprofile_operations *ops)
{
	init_ibs();
	create_arch_files = ops->create_files;
	ops->create_files = setup_ibs_files;

	if (boot_cpu_data.x86 == 0x15) {
		num_counters = AMD64_NUM_COUNTERS_CORE;
	} else {
		num_counters = AMD64_NUM_COUNTERS;
	}

	op_amd_spec.num_counters = num_counters;
	op_amd_spec.num_controls = num_counters;
	op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);

	return 0;
}

struct op_x86_model_spec op_amd_spec = {
	/* num_counters/num_controls filled in at runtime */
	.reserved		= MSR_AMD_EVENTSEL_RESERVED,
	.event_mask		= OP_EVENT_MASK,
	.init			= op_amd_init,
	.fill_in_addresses	= &op_amd_fill_in_addresses,
	.setup_ctrs		= &op_amd_setup_ctrs,
	.check_ctrs		= &op_amd_check_ctrs,
	.start			= &op_amd_start,
	.stop			= &op_amd_stop,
	.shutdown		= &op_amd_shutdown,
#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	.switch_ctrl		= &op_mux_switch_ctrl,
#endif
};
Commit	Line	Data
d4413732	1	/*
6852fd9b	2	* @file op_model_amd.c
bd87f1f0	3	* athlon / K7 / K8 / Family 10h model-specific MSR operations
1da177e4	4	*
ae735e99	5	* @remark Copyright 2002-2009 OProfile authors
1da177e4 LT	6	* @remark Read the file COPYING
	7	*
	8	* @author John Levon
	9	* @author Philippe Elie
	10	* @author Graydon Hoare
adf5ec0b	11	* @author Robert Richter <robert.richter@amd.com>
4d4036e0 JY	12	* @author Barry Kasindorf <barry.kasindorf@amd.com>
	13	* @author Jason Yeh <jason.yeh@amd.com>
	14	* @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
ae735e99	15	*/
1da177e4 LT	16
1da177e4 LT	17	#include <linux/oprofile.h>
56784f11 BK	18	#include <linux/device.h>
56784f11 BK	19	#include <linux/pci.h>
4d4036e0	20	#include <linux/percpu.h>
56784f11	21
1da177e4 LT	22	#include <asm/ptrace.h>
1da177e4 LT	23	#include <asm/msr.h>
3e4ff115	24	#include <asm/nmi.h>
013cfc50	25	#include <asm/apic.h>
64683da6	26	#include <asm/processor.h>
d4413732	27
1da177e4 LT	28	#include "op_x86_model.h"
	29	#include "op_counter.h"
	30
4d4036e0	31	#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
da169f5d	32	#define NUM_VIRT_COUNTERS 32
4d4036e0	33	#else
da169f5d	34	#define NUM_VIRT_COUNTERS 0
4d4036e0 JY	35	#endif
4d4036e0 JY	36
3370d358	37	#define OP_EVENT_MASK 0x0FFF
42399adb	38	#define OP_CTR_OVERFLOW (1ULL<<31)
3370d358 RR	39
3370d358 RR	40	#define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)\|(1ULL<<21))
1da177e4	41
da169f5d RR	42	static int num_counters;
da169f5d RR	43	static unsigned long reset_value[OP_MAX_COUNTER];
852402cc	44
c572ae4e RR	45	#define IBS_FETCH_SIZE 6
c572ae4e RR	46	#define IBS_OP_SIZE 12
56784f11	47
64683da6	48	static u32 ibs_caps;
56784f11	49
53b39e94	50	struct ibs_config {
56784f11 BK	51	unsigned long op_enabled;
	52	unsigned long fetch_enabled;
	53	unsigned long max_cnt_fetch;
	54	unsigned long max_cnt_op;
	55	unsigned long rand_en;
	56	unsigned long dispatched_ops;
25da6950	57	unsigned long branch_target;
56784f11 BK	58	};
56784f11 BK	59
53b39e94	60	struct ibs_state {
25da6950 RR	61	u64 ibs_op_ctl;
	62	int branch_target;
	63	unsigned long sample_size;
53b39e94 RR	64	};
	65
	66	static struct ibs_config ibs_config;
	67	static struct ibs_state ibs_state;
d4413732	68
ba52078e RR	69	/*
	70	* IBS randomization macros
	71	*/
	72	#define IBS_RANDOM_BITS 12
	73	#define IBS_RANDOM_MASK ((1ULL << IBS_RANDOM_BITS) - 1)
	74	#define IBS_RANDOM_MAXCNT_OFFSET (1ULL << (IBS_RANDOM_BITS - 5))
	75
f125be14 SS	76	/*
	77	* 16-bit Linear Feedback Shift Register (LFSR)
	78	*
	79	* 16 14 13 11
	80	* Feedback polynomial = X + X + X + X + 1
	81	*/
	82	static unsigned int lfsr_random(void)
	83	{
	84	static unsigned int lfsr_value = 0xF00D;
	85	unsigned int bit;
	86
	87	/* Compute next bit to shift in */
	88	bit = ((lfsr_value >> 0) ^
	89	(lfsr_value >> 2) ^
	90	(lfsr_value >> 3) ^
	91	(lfsr_value >> 5)) & 0x0001;
	92
	93	/* Advance to next register value */
	94	lfsr_value = (lfsr_value >> 1) \| (bit << 15);
	95
	96	return lfsr_value;
	97	}
	98
ba52078e RR	99	/*
	100	* IBS software randomization
	101	*
	102	* The IBS periodic op counter is randomized in software. The lower 12
	103	* bits of the 20 bit counter are randomized. IbsOpCurCnt is
	104	* initialized with a 12 bit random value.
	105	*/
	106	static inline u64 op_amd_randomize_ibs_op(u64 val)
	107	{
	108	unsigned int random = lfsr_random();
	109
	110	if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
	111	/*
	112	* Work around if the hw can not write to IbsOpCurCnt
	113	*
	114	* Randomize the lower 8 bits of the 16 bit
	115	* IbsOpMaxCnt [15:0] value in the range of -128 to
	116	* +127 by adding/subtracting an offset to the
	117	* maximum count (IbsOpMaxCnt).
	118	*
	119	* To avoid over or underflows and protect upper bits
	120	* starting at bit 16, the initial value for
	121	* IbsOpMaxCnt must fit in the range from 0x0081 to
	122	* 0xff80.
	123	*/
	124	val += (s8)(random >> 4);
	125	else
	126	val \|= (u64)(random & IBS_RANDOM_MASK) << 32;
	127
	128	return val;
	129	}
	130
4680e64a	131	static inline void
7939d2bf RR	132	op_amd_handle_ibs(struct pt_regs * const regs,
7939d2bf RR	133	struct op_msrs const * const msrs)
1da177e4	134	{
c572ae4e	135	u64 val, ctl;
1acda878	136	struct op_entry entry;
1da177e4	137
64683da6	138	if (!ibs_caps)
4680e64a	139	return;
1da177e4	140
7939d2bf	141	if (ibs_config.fetch_enabled) {
c572ae4e RR	142	rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
	143	if (ctl & IBS_FETCH_VAL) {
	144	rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
	145	oprofile_write_reserve(&entry, regs, val,
14f0ca8e	146	IBS_FETCH_CODE, IBS_FETCH_SIZE);
51563a0e RR	147	oprofile_add_data64(&entry, val);
51563a0e RR	148	oprofile_add_data64(&entry, ctl);
c572ae4e	149	rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
51563a0e	150	oprofile_add_data64(&entry, val);
14f0ca8e	151	oprofile_write_commit(&entry);
56784f11	152
fd13f6c8	153	/* reenable the IRQ */
a163b109	154	ctl &= ~(IBS_FETCH_VAL \| IBS_FETCH_CNT);
c572ae4e RR	155	ctl \|= IBS_FETCH_ENABLE;
c572ae4e RR	156	wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
56784f11 BK	157	}
	158	}
	159
7939d2bf	160	if (ibs_config.op_enabled) {
c572ae4e RR	161	rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
	162	if (ctl & IBS_OP_VAL) {
	163	rdmsrl(MSR_AMD64_IBSOPRIP, val);
25da6950 RR	164	oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE,
25da6950 RR	165	ibs_state.sample_size);
51563a0e	166	oprofile_add_data64(&entry, val);
c572ae4e	167	rdmsrl(MSR_AMD64_IBSOPDATA, val);
51563a0e	168	oprofile_add_data64(&entry, val);
c572ae4e	169	rdmsrl(MSR_AMD64_IBSOPDATA2, val);
51563a0e	170	oprofile_add_data64(&entry, val);
c572ae4e	171	rdmsrl(MSR_AMD64_IBSOPDATA3, val);
51563a0e	172	oprofile_add_data64(&entry, val);
c572ae4e	173	rdmsrl(MSR_AMD64_IBSDCLINAD, val);
51563a0e	174	oprofile_add_data64(&entry, val);
c572ae4e	175	rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
51563a0e	176	oprofile_add_data64(&entry, val);
25da6950 RR	177	if (ibs_state.branch_target) {
	178	rdmsrl(MSR_AMD64_IBSBRTARGET, val);
	179	oprofile_add_data(&entry, (unsigned long)val);
	180	}
14f0ca8e	181	oprofile_write_commit(&entry);
56784f11 BK	182
56784f11 BK	183	/* reenable the IRQ */
53b39e94	184	ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
c572ae4e	185	wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
56784f11 BK	186	}
56784f11 BK	187	}
1da177e4 LT	188	}
1da177e4 LT	189
90637595 RR	190	static inline void op_amd_start_ibs(void)
90637595 RR	191	{
c572ae4e	192	u64 val;
64683da6 RR	193
	194	if (!ibs_caps)
	195	return;
	196
53b39e94 RR	197	memset(&ibs_state, 0, sizeof(ibs_state));
53b39e94 RR	198
b47fad3b RR	199	/*
	200	* Note: Since the max count settings may out of range we
	201	* write back the actual used values so that userland can read
	202	* it.
	203	*/
	204
64683da6	205	if (ibs_config.fetch_enabled) {
b47fad3b RR	206	val = ibs_config.max_cnt_fetch >> 4;
	207	val = min(val, IBS_FETCH_MAX_CNT);
	208	ibs_config.max_cnt_fetch = val << 4;
c572ae4e RR	209	val \|= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
	210	val \|= IBS_FETCH_ENABLE;
	211	wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
90637595 RR	212	}
90637595 RR	213
64683da6	214	if (ibs_config.op_enabled) {
53b39e94	215	val = ibs_config.max_cnt_op >> 4;
ba52078e RR	216	if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
	217	/*
	218	* IbsOpCurCnt not supported. See
	219	* op_amd_randomize_ibs_op() for details.
	220	*/
53b39e94	221	val = clamp(val, 0x0081ULL, 0xFF80ULL);
b47fad3b	222	ibs_config.max_cnt_op = val << 4;
ba52078e RR	223	} else {
	224	/*
	225	* The start value is randomized with a
	226	* positive offset, we need to compensate it
	227	* with the half of the randomized range. Also
	228	* avoid underflows.
	229	*/
b47fad3b RR	230	val += IBS_RANDOM_MAXCNT_OFFSET;
	231	if (ibs_caps & IBS_CAPS_OPCNTEXT)
	232	val = min(val, IBS_OP_MAX_CNT_EXT);
	233	else
	234	val = min(val, IBS_OP_MAX_CNT);
	235	ibs_config.max_cnt_op =
	236	(val - IBS_RANDOM_MAXCNT_OFFSET) << 4;
ba52078e	237	}
b47fad3b	238	val = ((val & ~IBS_OP_MAX_CNT) << 4) \| (val & IBS_OP_MAX_CNT);
53b39e94 RR	239	val \|= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
	240	val \|= IBS_OP_ENABLE;
	241	ibs_state.ibs_op_ctl = val;
25da6950 RR	242	ibs_state.sample_size = IBS_OP_SIZE;
	243	if (ibs_config.branch_target) {
	244	ibs_state.branch_target = 1;
	245	ibs_state.sample_size++;
ba52078e	246	}
53b39e94	247	val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
c572ae4e	248	wrmsrl(MSR_AMD64_IBSOPCTL, val);
90637595 RR	249	}
	250	}
	251
	252	static void op_amd_stop_ibs(void)
	253	{
64683da6 RR	254	if (!ibs_caps)
	255	return;
	256
	257	if (ibs_config.fetch_enabled)
90637595	258	/* clear max count and enable */
c572ae4e	259	wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
90637595	260
64683da6	261	if (ibs_config.op_enabled)
90637595	262	/* clear max count and enable */
c572ae4e	263	wrmsrl(MSR_AMD64_IBSOPCTL, 0);
90637595 RR	264	}
90637595 RR	265
da759fe5 RR	266	#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	267
	268	static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
	269	struct op_msrs const * const msrs)
	270	{
	271	u64 val;
	272	int i;
	273
	274	/* enable active counters */
da169f5d	275	for (i = 0; i < num_counters; ++i) {
da759fe5 RR	276	int virt = op_x86_phys_to_virt(i);
	277	if (!reset_value[virt])
	278	continue;
	279	rdmsrl(msrs->controls[i].addr, val);
	280	val &= model->reserved;
	281	val \|= op_x86_get_ctrl(model, &counter_config[virt]);
	282	wrmsrl(msrs->controls[i].addr, val);
	283	}
	284	}
	285
	286	#endif
	287
	288	/* functions for op_amd_spec */
	289
	290	static void op_amd_shutdown(struct op_msrs const * const msrs)
	291	{
	292	int i;
	293
da169f5d	294	for (i = 0; i < num_counters; ++i) {
da759fe5 RR	295	if (!msrs->counters[i].addr)
	296	continue;
	297	release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
	298	release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
	299	}
	300	}
	301
	302	static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
	303	{
	304	int i;
	305
da169f5d	306	for (i = 0; i < num_counters; i++) {
da759fe5 RR	307	if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
	308	goto fail;
	309	if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
	310	release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
	311	goto fail;
	312	}
	313	/* both registers must be reserved */
b1dc3c48	314	if (num_counters == AMD64_NUM_COUNTERS_CORE) {
da169f5d RR	315	msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1);
	316	msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1);
	317	} else {
	318	msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
	319	msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
	320	}
da759fe5 RR	321	continue;
	322	fail:
	323	if (!counter_config[i].enabled)
	324	continue;
	325	op_x86_warn_reserved(i);
	326	op_amd_shutdown(msrs);
	327	return -EBUSY;
	328	}
	329
	330	return 0;
	331	}
	332
	333	static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
	334	struct op_msrs const * const msrs)
	335	{
	336	u64 val;
	337	int i;
	338
	339	/* setup reset_value */
da169f5d	340	for (i = 0; i < OP_MAX_COUNTER; ++i) {
da759fe5 RR	341	if (counter_config[i].enabled
	342	&& msrs->counters[op_x86_virt_to_phys(i)].addr)
	343	reset_value[i] = counter_config[i].count;
	344	else
	345	reset_value[i] = 0;
	346	}
	347
	348	/* clear all counters */
da169f5d	349	for (i = 0; i < num_counters; ++i) {
da759fe5 RR	350	if (!msrs->controls[i].addr)
	351	continue;
	352	rdmsrl(msrs->controls[i].addr, val);
	353	if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
	354	op_x86_warn_in_use(i);
	355	val &= model->reserved;
	356	wrmsrl(msrs->controls[i].addr, val);
	357	/*
	358	* avoid a false detection of ctr overflows in NMI
	359	* handler
	360	*/
	361	wrmsrl(msrs->counters[i].addr, -1LL);
	362	}
	363
	364	/* enable active counters */
da169f5d	365	for (i = 0; i < num_counters; ++i) {
da759fe5 RR	366	int virt = op_x86_phys_to_virt(i);
	367	if (!reset_value[virt])
	368	continue;
	369
	370	/* setup counter registers */
	371	wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
	372
	373	/* setup control registers */
	374	rdmsrl(msrs->controls[i].addr, val);
	375	val &= model->reserved;
	376	val \|= op_x86_get_ctrl(model, &counter_config[virt]);
	377	wrmsrl(msrs->controls[i].addr, val);
	378	}
	379	}
	380
7939d2bf RR	381	static int op_amd_check_ctrs(struct pt_regs * const regs,
	382	struct op_msrs const * const msrs)
	383	{
42399adb	384	u64 val;
7939d2bf RR	385	int i;
7939d2bf RR	386
da169f5d	387	for (i = 0; i < num_counters; ++i) {
d8471ad3 RR	388	int virt = op_x86_phys_to_virt(i);
d8471ad3 RR	389	if (!reset_value[virt])
7939d2bf	390	continue;
42399adb RR	391	rdmsrl(msrs->counters[i].addr, val);
	392	/* bit is clear if overflowed: */
	393	if (val & OP_CTR_OVERFLOW)
	394	continue;
d8471ad3 RR	395	oprofile_add_sample(regs, virt);
d8471ad3 RR	396	wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
7939d2bf RR	397	}
	398
	399	op_amd_handle_ibs(regs, msrs);
	400
	401	/* See op_model_ppro.c */
	402	return 1;
	403	}
d4413732	404
6657fe4f	405	static void op_amd_start(struct op_msrs const * const msrs)
1da177e4	406	{
dea3766c	407	u64 val;
1da177e4	408	int i;
4d4036e0	409
da169f5d	410	for (i = 0; i < num_counters; ++i) {
d8471ad3 RR	411	if (!reset_value[op_x86_phys_to_virt(i)])
	412	continue;
	413	rdmsrl(msrs->controls[i].addr, val);
bb1165d6	414	val \|= ARCH_PERFMON_EVENTSEL_ENABLE;
d8471ad3	415	wrmsrl(msrs->controls[i].addr, val);
1da177e4	416	}
852402cc	417
90637595	418	op_amd_start_ibs();
1da177e4 LT	419	}
1da177e4 LT	420
6657fe4f	421	static void op_amd_stop(struct op_msrs const * const msrs)
1da177e4	422	{
dea3766c	423	u64 val;
1da177e4 LT	424	int i;
1da177e4 LT	425
fd13f6c8 RR	426	/*
	427	* Subtle: stop on all counters to avoid race with setting our
	428	* pm callback
	429	*/
da169f5d	430	for (i = 0; i < num_counters; ++i) {
d8471ad3	431	if (!reset_value[op_x86_phys_to_virt(i)])
cb9c448c	432	continue;
dea3766c	433	rdmsrl(msrs->controls[i].addr, val);
bb1165d6	434	val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
dea3766c	435	wrmsrl(msrs->controls[i].addr, val);
1da177e4	436	}
56784f11	437
90637595	438	op_amd_stop_ibs();
1da177e4 LT	439	}
1da177e4 LT	440
c7c25802 RR	441	/*
	442	* check and reserve APIC extended interrupt LVT offset for IBS if
	443	* available
c7c25802 RR	444	*/
c7c25802 RR	445
bae663bc	446	static void init_ibs(void)
56784f11	447	{
c7c25802	448	ibs_caps = get_ibs_caps();
3d2606f4	449
64683da6	450	if (!ibs_caps)
3d2606f4 RR	451	return;
3d2606f4 RR	452
3d2606f4	453	printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps);
56784f11 BK	454	}
56784f11 BK	455
ef7bca14	456	static int (create_arch_files)(struct dentry root);
270d3e1a	457
ef7bca14	458	static int setup_ibs_files(struct dentry *root)
56784f11	459	{
56784f11	460	struct dentry *dir;
270d3e1a RR	461	int ret = 0;
	462
	463	/* architecture specific files */
	464	if (create_arch_files)
ef7bca14	465	ret = create_arch_files(root);
270d3e1a RR	466
	467	if (ret)
	468	return ret;
56784f11	469
64683da6	470	if (!ibs_caps)
270d3e1a RR	471	return ret;
	472
	473	/* model specific files */
56784f11 BK	474
56784f11 BK	475	/* setup some reasonable defaults */
25da6950	476	memset(&ibs_config, 0, sizeof(ibs_config));
56784f11	477	ibs_config.max_cnt_fetch = 250000;
56784f11	478	ibs_config.max_cnt_op = 250000;
2d55a478	479
4ac945f0	480	if (ibs_caps & IBS_CAPS_FETCHSAM) {
ecde2823	481	dir = oprofilefs_mkdir(root, "ibs_fetch");
6af4ea0b	482	oprofilefs_create_ulong(dir, "enable",
4ac945f0	483	&ibs_config.fetch_enabled);
6af4ea0b	484	oprofilefs_create_ulong(dir, "max_count",
4ac945f0	485	&ibs_config.max_cnt_fetch);
6af4ea0b	486	oprofilefs_create_ulong(dir, "rand_enable",
4ac945f0 RR	487	&ibs_config.rand_en);
	488	}
	489
	490	if (ibs_caps & IBS_CAPS_OPSAM) {
ecde2823	491	dir = oprofilefs_mkdir(root, "ibs_op");
6af4ea0b	492	oprofilefs_create_ulong(dir, "enable",
4ac945f0	493	&ibs_config.op_enabled);
6af4ea0b	494	oprofilefs_create_ulong(dir, "max_count",
4ac945f0 RR	495	&ibs_config.max_cnt_op);
4ac945f0 RR	496	if (ibs_caps & IBS_CAPS_OPCNT)
6af4ea0b	497	oprofilefs_create_ulong(dir, "dispatched_ops",
4ac945f0	498	&ibs_config.dispatched_ops);
25da6950	499	if (ibs_caps & IBS_CAPS_BRNTRGT)
6af4ea0b	500	oprofilefs_create_ulong(dir, "branch_target",
25da6950	501	&ibs_config.branch_target);
4ac945f0	502	}
fc2bd734 RR	503
fc2bd734 RR	504	return 0;
56784f11 BK	505	}
56784f11 BK	506
da169f5d RR	507	struct op_x86_model_spec op_amd_spec;
da169f5d RR	508
adf5ec0b RR	509	static int op_amd_init(struct oprofile_operations *ops)
adf5ec0b RR	510	{
bae663bc	511	init_ibs();
270d3e1a RR	512	create_arch_files = ops->create_files;
270d3e1a RR	513	ops->create_files = setup_ibs_files;
da169f5d RR	514
da169f5d RR	515	if (boot_cpu_data.x86 == 0x15) {
b1dc3c48	516	num_counters = AMD64_NUM_COUNTERS_CORE;
da169f5d	517	} else {
ee5789db	518	num_counters = AMD64_NUM_COUNTERS;
da169f5d RR	519	}
	520
	521	op_amd_spec.num_counters = num_counters;
	522	op_amd_spec.num_controls = num_counters;
	523	op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
	524
adf5ec0b RR	525	return 0;
	526	}
	527
259a83a8	528	struct op_x86_model_spec op_amd_spec = {
da169f5d	529	/* num_counters/num_controls filled in at runtime */
3370d358 RR	530	.reserved = MSR_AMD_EVENTSEL_RESERVED,
	531	.event_mask = OP_EVENT_MASK,
	532	.init = op_amd_init,
c92960fc RR	533	.fill_in_addresses = &op_amd_fill_in_addresses,
	534	.setup_ctrs = &op_amd_setup_ctrs,
	535	.check_ctrs = &op_amd_check_ctrs,
	536	.start = &op_amd_start,
	537	.stop = &op_amd_stop,
3370d358	538	.shutdown = &op_amd_shutdown,
4d4036e0	539	#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
7e7478c6	540	.switch_ctrl = &op_mux_switch_ctrl,
4d4036e0	541	#endif
1da177e4	542	};