[deliverable/linux.git] / arch / ia64 / kernel / patch.c

/*
 * Instruction-patching support.
 *
 * Copyright (C) 2003 Hewlett-Packard Co
 *	David Mosberger-Tang <davidm@hpl.hp.com>
 */
#include <linux/init.h>
#include <linux/string.h>

#include <asm/patch.h>
#include <asm/processor.h>
#include <asm/sections.h>
#include <asm/unistd.h>

/*
 * This was adapted from code written by Tony Luck:
 *
 * The 64-bit value in a "movl reg=value" is scattered between the two words of the bundle
 * like this:
 *
 * 6  6         5         4         3         2         1
 * 3210987654321098765432109876543210987654321098765432109876543210
 * ABBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCDEEEEEFFFFFFFFFGGGGGGG
 *
 * CCCCCCCCCCCCCCCCCCxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 * xxxxAFFFFFFFFFEEEEEDxGGGGGGGxxxxxxxxxxxxxBBBBBBBBBBBBBBBBBBBBBBB
 */
static u64
get_imm64 (u64 insn_addr)
{
	u64 *p = (u64 *) (insn_addr & -16);	/* mask out slot number */

	return ( (p[1] & 0x0800000000000000UL) << 4)  | /*A*/
		((p[1] & 0x00000000007fffffUL) << 40) | /*B*/
		((p[0] & 0xffffc00000000000UL) >> 24) | /*C*/
		((p[1] & 0x0000100000000000UL) >> 23) | /*D*/
		((p[1] & 0x0003e00000000000UL) >> 29) | /*E*/
		((p[1] & 0x07fc000000000000UL) >> 43) | /*F*/
		((p[1] & 0x000007f000000000UL) >> 36);  /*G*/
}

/* Patch instruction with "val" where "mask" has 1 bits. */
void
ia64_patch (u64 insn_addr, u64 mask, u64 val)
{
	u64 m0, m1, v0, v1, b0, b1, *b = (u64 *) (insn_addr & -16);
#	define insn_mask ((1UL << 41) - 1)
	unsigned long shift;

	b0 = b[0]; b1 = b[1];
	shift = 5 + 41 * (insn_addr % 16); /* 5 bits of template, then 3 x 41-bit instructions */
	if (shift >= 64) {
		m1 = mask << (shift - 64);
		v1 = val << (shift - 64);
	} else {
		m0 = mask << shift; m1 = mask >> (64 - shift);
		v0 = val  << shift; v1 = val >> (64 - shift);
		b[0] = (b0 & ~m0) | (v0 & m0);
	}
	b[1] = (b1 & ~m1) | (v1 & m1);
}

void
ia64_patch_imm64 (u64 insn_addr, u64 val)
{
	/* The assembler may generate offset pointing to either slot 1
	   or slot 2 for a long (2-slot) instruction, occupying slots 1
	   and 2.  */
  	insn_addr &= -16UL;
	ia64_patch(insn_addr + 2,
		   0x01fffefe000UL, (  ((val & 0x8000000000000000UL) >> 27) /* bit 63 -> 36 */
				     | ((val & 0x0000000000200000UL) <<  0) /* bit 21 -> 21 */
				     | ((val & 0x00000000001f0000UL) <<  6) /* bit 16 -> 22 */
				     | ((val & 0x000000000000ff80UL) << 20) /* bit  7 -> 27 */
				     | ((val & 0x000000000000007fUL) << 13) /* bit  0 -> 13 */));
	ia64_patch(insn_addr + 1, 0x1ffffffffffUL, val >> 22);
}

void
ia64_patch_imm60 (u64 insn_addr, u64 val)
{
	/* The assembler may generate offset pointing to either slot 1
	   or slot 2 for a long (2-slot) instruction, occupying slots 1
	   and 2.  */
  	insn_addr &= -16UL;
	ia64_patch(insn_addr + 2,
		   0x011ffffe000UL, (  ((val & 0x0800000000000000UL) >> 23) /* bit 59 -> 36 */
				     | ((val & 0x00000000000fffffUL) << 13) /* bit  0 -> 13 */));
	ia64_patch(insn_addr + 1, 0x1fffffffffcUL, val >> 18);
}

/*
 * We need sometimes to load the physical address of a kernel
 * object.  Often we can convert the virtual address to physical
 * at execution time, but sometimes (either for performance reasons
 * or during error recovery) we cannot to this.  Patch the marked
 * bundles to load the physical address.
 */
void __init
ia64_patch_vtop (unsigned long start, unsigned long end)
{
	s32 *offp = (s32 *) start;
	u64 ip;

	while (offp < (s32 *) end) {
		ip = (u64) offp + *offp;

		/* replace virtual address with corresponding physical address: */
		ia64_patch_imm64(ip, ia64_tpa(get_imm64(ip)));
		ia64_fc((void *) ip);
		++offp;
	}
	ia64_sync_i();
	ia64_srlz_i();
}

/*
 * Disable the RSE workaround by turning the conditional branch
 * that we tagged in each place the workaround was used into an
 * unconditional branch.
 */
void __init
ia64_patch_rse (unsigned long start, unsigned long end)
{
	s32 *offp = (s32 *) start;
	u64 ip, *b;

	while (offp < (s32 *) end) {
		ip = (u64) offp + *offp;

		b = (u64 *)(ip & -16);
		b[1] &= ~0xf800000L;
		ia64_fc((void *) ip);
		++offp;
	}
	ia64_sync_i();
	ia64_srlz_i();
}

void __init
ia64_patch_mckinley_e9 (unsigned long start, unsigned long end)
{
	static int first_time = 1;
	int need_workaround;
	s32 *offp = (s32 *) start;
	u64 *wp;

	need_workaround = (local_cpu_data->family == 0x1f && local_cpu_data->model == 0);

	if (first_time) {
		first_time = 0;
		if (need_workaround)
			printk(KERN_INFO "Leaving McKinley Errata 9 workaround enabled\n");
	}
	if (need_workaround)
		return;

	while (offp < (s32 *) end) {
		wp = (u64 *) ia64_imva((char *) offp + *offp);
		wp[0] = 0x0000000100000011UL; /* nop.m 0; nop.i 0; br.ret.sptk.many b6 */
		wp[1] = 0x0084006880000200UL;
		wp[2] = 0x0000000100000000UL; /* nop.m 0; nop.i 0; nop.i 0 */
		wp[3] = 0x0004000000000200UL;
		ia64_fc(wp); ia64_fc(wp + 2);
		++offp;
	}
	ia64_sync_i();
	ia64_srlz_i();
}

static void __init
patch_fsyscall_table (unsigned long start, unsigned long end)
{
	extern unsigned long fsyscall_table[NR_syscalls];
	s32 *offp = (s32 *) start;
	u64 ip;

	while (offp < (s32 *) end) {
		ip = (u64) ia64_imva((char *) offp + *offp);
		ia64_patch_imm64(ip, (u64) fsyscall_table);
		ia64_fc((void *) ip);
		++offp;
	}
	ia64_sync_i();
	ia64_srlz_i();
}

static void __init
patch_brl_fsys_bubble_down (unsigned long start, unsigned long end)
{
	extern char fsys_bubble_down[];
	s32 *offp = (s32 *) start;
	u64 ip;

	while (offp < (s32 *) end) {
		ip = (u64) offp + *offp;
		ia64_patch_imm60((u64) ia64_imva((void *) ip),
				 (u64) (fsys_bubble_down - (ip & -16)) / 16);
		ia64_fc((void *) ip);
		++offp;
	}
	ia64_sync_i();
	ia64_srlz_i();
}

void __init
ia64_patch_gate (void)
{
#	define START(name)	((unsigned long) __start_gate_##name##_patchlist)
#	define END(name)	((unsigned long)__end_gate_##name##_patchlist)

	patch_fsyscall_table(START(fsyscall), END(fsyscall));
	patch_brl_fsys_bubble_down(START(brl_fsys_bubble_down), END(brl_fsys_bubble_down));
	ia64_patch_vtop(START(vtop), END(vtop));
	ia64_patch_mckinley_e9(START(mckinley_e9), END(mckinley_e9));
}

void ia64_patch_phys_stack_reg(unsigned long val)
{
	s32 * offp = (s32 *) __start___phys_stack_reg_patchlist;
	s32 * end = (s32 *) __end___phys_stack_reg_patchlist;
	u64 ip, mask, imm;

	/* see instruction format A4: adds r1 = imm13, r3 */
	mask = (0x3fUL << 27) | (0x7f << 13);
	imm = (((val >> 7) & 0x3f) << 27) | (val & 0x7f) << 13;

	while (offp < end) {
		ip = (u64) offp + *offp;
		ia64_patch(ip, mask, imm);
		ia64_fc((void *)ip);
		++offp;
	}
	ia64_sync_i();
	ia64_srlz_i();
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Instruction-patching support.
	3	*
	4	* Copyright (C) 2003 Hewlett-Packard Co
	5	* David Mosberger-Tang <davidm@hpl.hp.com>
	6	*/
	7	#include <linux/init.h>
	8	#include <linux/string.h>
	9
	10	#include <asm/patch.h>
	11	#include <asm/processor.h>
	12	#include <asm/sections.h>
1da177e4 LT	13	#include <asm/unistd.h>
	14
	15	/*
	16	* This was adapted from code written by Tony Luck:
	17	*
	18	* The 64-bit value in a "movl reg=value" is scattered between the two words of the bundle
	19	* like this:
	20	*
	21	* 6 6 5 4 3 2 1
	22	* 3210987654321098765432109876543210987654321098765432109876543210
	23	* ABBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCDEEEEEFFFFFFFFFGGGGGGG
	24	*
	25	* CCCCCCCCCCCCCCCCCCxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
	26	* xxxxAFFFFFFFFFEEEEEDxGGGGGGGxxxxxxxxxxxxxBBBBBBBBBBBBBBBBBBBBBBB
	27	*/
	28	static u64
	29	get_imm64 (u64 insn_addr)
	30	{
	31	u64 p = (u64 ) (insn_addr & -16); /* mask out slot number */
	32
	33	return ( (p[1] & 0x0800000000000000UL) << 4) \| /A/
	34	((p[1] & 0x00000000007fffffUL) << 40) \| /B/
	35	((p[0] & 0xffffc00000000000UL) >> 24) \| /C/
	36	((p[1] & 0x0000100000000000UL) >> 23) \| /D/
	37	((p[1] & 0x0003e00000000000UL) >> 29) \| /E/
	38	((p[1] & 0x07fc000000000000UL) >> 43) \| /F/
	39	((p[1] & 0x000007f000000000UL) >> 36); /G/
	40	}
	41
	42	/* Patch instruction with "val" where "mask" has 1 bits. */
	43	void
	44	ia64_patch (u64 insn_addr, u64 mask, u64 val)
	45	{
	46	u64 m0, m1, v0, v1, b0, b1, b = (u64 ) (insn_addr & -16);
	47	# define insn_mask ((1UL << 41) - 1)
	48	unsigned long shift;
	49
	50	b0 = b[0]; b1 = b[1];
	51	shift = 5 + 41 * (insn_addr % 16); /* 5 bits of template, then 3 x 41-bit instructions */
	52	if (shift >= 64) {
	53	m1 = mask << (shift - 64);
	54	v1 = val << (shift - 64);
	55	} else {
	56	m0 = mask << shift; m1 = mask >> (64 - shift);
	57	v0 = val << shift; v1 = val >> (64 - shift);
	58	b[0] = (b0 & ~m0) \| (v0 & m0);
	59	}
	60	b[1] = (b1 & ~m1) \| (v1 & m1);
	61	}
	62
	63	void
	64	ia64_patch_imm64 (u64 insn_addr, u64 val)
	65	{
9c184a07 L	66	/* The assembler may generate offset pointing to either slot 1
	67	or slot 2 for a long (2-slot) instruction, occupying slots 1
	68	and 2. */
	69	insn_addr &= -16UL;
	70	ia64_patch(insn_addr + 2,
1da177e4 LT	71	0x01fffefe000UL, ( ((val & 0x8000000000000000UL) >> 27) /* bit 63 -> 36 */
	72	\| ((val & 0x0000000000200000UL) << 0) /* bit 21 -> 21 */
	73	\| ((val & 0x00000000001f0000UL) << 6) /* bit 16 -> 22 */
	74	\| ((val & 0x000000000000ff80UL) << 20) /* bit 7 -> 27 */
	75	\| ((val & 0x000000000000007fUL) << 13) /* bit 0 -> 13 */));
9c184a07	76	ia64_patch(insn_addr + 1, 0x1ffffffffffUL, val >> 22);
1da177e4 LT	77	}
	78
	79	void
	80	ia64_patch_imm60 (u64 insn_addr, u64 val)
	81	{
9c184a07 L	82	/* The assembler may generate offset pointing to either slot 1
	83	or slot 2 for a long (2-slot) instruction, occupying slots 1
	84	and 2. */
	85	insn_addr &= -16UL;
	86	ia64_patch(insn_addr + 2,
1da177e4 LT	87	0x011ffffe000UL, ( ((val & 0x0800000000000000UL) >> 23) /* bit 59 -> 36 */
1da177e4 LT	88	\| ((val & 0x00000000000fffffUL) << 13) /* bit 0 -> 13 */));
9c184a07	89	ia64_patch(insn_addr + 1, 0x1fffffffffcUL, val >> 18);
1da177e4 LT	90	}
	91
	92	/*
	93	* We need sometimes to load the physical address of a kernel
	94	* object. Often we can convert the virtual address to physical
	95	* at execution time, but sometimes (either for performance reasons
	96	* or during error recovery) we cannot to this. Patch the marked
	97	* bundles to load the physical address.
	98	*/
	99	void __init
	100	ia64_patch_vtop (unsigned long start, unsigned long end)
	101	{
	102	s32 offp = (s32 ) start;
	103	u64 ip;
	104
	105	while (offp < (s32 *) end) {
	106	ip = (u64) offp + *offp;
	107
	108	/* replace virtual address with corresponding physical address: */
	109	ia64_patch_imm64(ip, ia64_tpa(get_imm64(ip)));
	110	ia64_fc((void *) ip);
	111	++offp;
	112	}
	113	ia64_sync_i();
	114	ia64_srlz_i();
	115	}
	116
4dcc29e1 TL	117	/*
	118	* Disable the RSE workaround by turning the conditional branch
	119	* that we tagged in each place the workaround was used into an
	120	* unconditional branch.
	121	*/
	122	void __init
	123	ia64_patch_rse (unsigned long start, unsigned long end)
	124	{
	125	s32 offp = (s32 ) start;
	126	u64 ip, *b;
	127
	128	while (offp < (s32 *) end) {
	129	ip = (u64) offp + *offp;
	130
	131	b = (u64 *)(ip & -16);
	132	b[1] &= ~0xf800000L;
	133	ia64_fc((void *) ip);
	134	++offp;
	135	}
	136	ia64_sync_i();
	137	ia64_srlz_i();
	138	}
	139
914a4ea4	140	void __init
1da177e4 LT	141	ia64_patch_mckinley_e9 (unsigned long start, unsigned long end)
	142	{
	143	static int first_time = 1;
	144	int need_workaround;
	145	s32 offp = (s32 ) start;
	146	u64 *wp;
	147
	148	need_workaround = (local_cpu_data->family == 0x1f && local_cpu_data->model == 0);
	149
	150	if (first_time) {
	151	first_time = 0;
	152	if (need_workaround)
	153	printk(KERN_INFO "Leaving McKinley Errata 9 workaround enabled\n");
1da177e4 LT	154	}
	155	if (need_workaround)
	156	return;
	157
	158	while (offp < (s32 *) end) {
	159	wp = (u64 ) ia64_imva((char ) offp + *offp);
4fe01c68 HS	160	wp[0] = 0x0000000100000011UL; /* nop.m 0; nop.i 0; br.ret.sptk.many b6 */
	161	wp[1] = 0x0084006880000200UL;
	162	wp[2] = 0x0000000100000000UL; /* nop.m 0; nop.i 0; nop.i 0 */
	163	wp[3] = 0x0004000000000200UL;
1da177e4 LT	164	ia64_fc(wp); ia64_fc(wp + 2);
	165	++offp;
	166	}
	167	ia64_sync_i();
	168	ia64_srlz_i();
	169	}
	170
914a4ea4	171	static void __init
1da177e4 LT	172	patch_fsyscall_table (unsigned long start, unsigned long end)
1da177e4 LT	173	{
e55645ec	174	extern unsigned long fsyscall_table[NR_syscalls];
1da177e4 LT	175	s32 offp = (s32 ) start;
	176	u64 ip;
	177
	178	while (offp < (s32 *) end) {
	179	ip = (u64) ia64_imva((char ) offp + offp);
e55645ec	180	ia64_patch_imm64(ip, (u64) fsyscall_table);
1da177e4 LT	181	ia64_fc((void *) ip);
	182	++offp;
	183	}
	184	ia64_sync_i();
	185	ia64_srlz_i();
	186	}
	187
914a4ea4	188	static void __init
1da177e4 LT	189	patch_brl_fsys_bubble_down (unsigned long start, unsigned long end)
1da177e4 LT	190	{
e55645ec	191	extern char fsys_bubble_down[];
1da177e4 LT	192	s32 offp = (s32 ) start;
	193	u64 ip;
	194
	195	while (offp < (s32 *) end) {
	196	ip = (u64) offp + *offp;
	197	ia64_patch_imm60((u64) ia64_imva((void *) ip),
	198	(u64) (fsys_bubble_down - (ip & -16)) / 16);
	199	ia64_fc((void *) ip);
	200	++offp;
	201	}
	202	ia64_sync_i();
	203	ia64_srlz_i();
	204	}
	205
914a4ea4	206	void __init
1da177e4 LT	207	ia64_patch_gate (void)
1da177e4 LT	208	{
e55645ec LR	209	# define START(name) ((unsigned long) __start_gate_##name##_patchlist)
e55645ec LR	210	# define END(name) ((unsigned long)__end_gate_##name##_patchlist)
1da177e4	211
e55645ec LR	212	patch_fsyscall_table(START(fsyscall), END(fsyscall));
	213	patch_brl_fsys_bubble_down(START(brl_fsys_bubble_down), END(brl_fsys_bubble_down));
	214	ia64_patch_vtop(START(vtop), END(vtop));
	215	ia64_patch_mckinley_e9(START(mckinley_e9), END(mckinley_e9));
1da177e4	216	}
a0776ec8 CK	217
	218	void ia64_patch_phys_stack_reg(unsigned long val)
	219	{
	220	s32 * offp = (s32 *) __start___phys_stack_reg_patchlist;
	221	s32 * end = (s32 *) __end___phys_stack_reg_patchlist;
	222	u64 ip, mask, imm;
	223
	224	/* see instruction format A4: adds r1 = imm13, r3 */
	225	mask = (0x3fUL << 27) \| (0x7f << 13);
	226	imm = (((val >> 7) & 0x3f) << 27) \| (val & 0x7f) << 13;
	227
	228	while (offp < end) {
	229	ip = (u64) offp + *offp;
	230	ia64_patch(ip, mask, imm);
7120569c	231	ia64_fc((void *)ip);
a0776ec8 CK	232	++offp;
	233	}
	234	ia64_sync_i();
	235	ia64_srlz_i();
	236	}