[deliverable/linux.git] / arch / arm / lib / memmove.S

/*
 *  linux/arch/arm/lib/memmove.S
 *
 *  Author:	Nicolas Pitre
 *  Created:	Sep 28, 2005
 *  Copyright:	(C) MontaVista Software Inc.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License version 2 as
 *  published by the Free Software Foundation.
 */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>
#include <asm/export.h>

		.text

/*
 * Prototype: void *memmove(void *dest, const void *src, size_t n);
 *
 * Note:
 *
 * If the memory regions don't overlap, we simply branch to memcpy which is
 * normally a bit faster. Otherwise the copy is done going downwards.  This
 * is a transposition of the code from copy_template.S but with the copy
 * occurring in the opposite direction.
 */

ENTRY(memmove)
	UNWIND(	.fnstart			)

		subs	ip, r0, r1
		cmphi	r2, ip
		bls	memcpy

		stmfd	sp!, {r0, r4, lr}
	UNWIND(	.fnend				)

	UNWIND(	.fnstart			)
	UNWIND(	.save	{r0, r4, lr}		) @ in first stmfd block
		add	r1, r1, r2
		add	r0, r0, r2
		subs	r2, r2, #4
		blt	8f
		ands	ip, r0, #3
	PLD(	pld	[r1, #-4]		)
		bne	9f
		ands	ip, r1, #3
		bne	10f

1:		subs	r2, r2, #(28)
		stmfd	sp!, {r5 - r8}
	UNWIND(	.fnend				)

	UNWIND(	.fnstart			)
	UNWIND(	.save	{r0, r4, lr}		)
	UNWIND(	.save	{r5 - r8}		) @ in second stmfd block
		blt	5f

	CALGN(	ands	ip, r0, #31		)
	CALGN(	sbcnes	r4, ip, r2		)  @ C is always set here
	CALGN(	bcs	2f			)
	CALGN(	adr	r4, 6f			)
	CALGN(	subs	r2, r2, ip		)  @ C is set here
	CALGN(	rsb	ip, ip, #32		)
	CALGN(	add	pc, r4, ip		)

	PLD(	pld	[r1, #-4]		)
2:	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #-32]		)
	PLD(	blt	4f			)
	PLD(	pld	[r1, #-64]		)
	PLD(	pld	[r1, #-96]		)

3:	PLD(	pld	[r1, #-128]		)
4:		ldmdb	r1!, {r3, r4, r5, r6, r7, r8, ip, lr}
		subs	r2, r2, #32
		stmdb	r0!, {r3, r4, r5, r6, r7, r8, ip, lr}
		bge	3b
	PLD(	cmn	r2, #96			)
	PLD(	bge	4b			)

5:		ands	ip, r2, #28
		rsb	ip, ip, #32
		addne	pc, pc, ip		@ C is always clear here
		b	7f
6:		W(nop)
		W(ldr)	r3, [r1, #-4]!
		W(ldr)	r4, [r1, #-4]!
		W(ldr)	r5, [r1, #-4]!
		W(ldr)	r6, [r1, #-4]!
		W(ldr)	r7, [r1, #-4]!
		W(ldr)	r8, [r1, #-4]!
		W(ldr)	lr, [r1, #-4]!

		add	pc, pc, ip
		nop
		W(nop)
		W(str)	r3, [r0, #-4]!
		W(str)	r4, [r0, #-4]!
		W(str)	r5, [r0, #-4]!
		W(str)	r6, [r0, #-4]!
		W(str)	r7, [r0, #-4]!
		W(str)	r8, [r0, #-4]!
		W(str)	lr, [r0, #-4]!

	CALGN(	bcs	2b			)

7:		ldmfd	sp!, {r5 - r8}
	UNWIND(	.fnend				) @ end of second stmfd block

	UNWIND(	.fnstart			)
	UNWIND(	.save	{r0, r4, lr}		) @ still in first stmfd block

8:		movs	r2, r2, lsl #31
		ldrneb	r3, [r1, #-1]!
		ldrcsb	r4, [r1, #-1]!
		ldrcsb	ip, [r1, #-1]
		strneb	r3, [r0, #-1]!
		strcsb	r4, [r0, #-1]!
		strcsb	ip, [r0, #-1]
		ldmfd	sp!, {r0, r4, pc}

9:		cmp	ip, #2
		ldrgtb	r3, [r1, #-1]!
		ldrgeb	r4, [r1, #-1]!
		ldrb	lr, [r1, #-1]!
		strgtb	r3, [r0, #-1]!
		strgeb	r4, [r0, #-1]!
		subs	r2, r2, ip
		strb	lr, [r0, #-1]!
		blt	8b
		ands	ip, r1, #3
		beq	1b

10:		bic	r1, r1, #3
		cmp	ip, #2
		ldr	r3, [r1, #0]
		beq	17f
		blt	18f
	UNWIND(	.fnend				)


		.macro	backward_copy_shift push pull

	UNWIND(	.fnstart			)
	UNWIND(	.save	{r0, r4, lr}		) @ still in first stmfd block
		subs	r2, r2, #28
		blt	14f

	CALGN(	ands	ip, r0, #31		)
	CALGN(	sbcnes	r4, ip, r2		)  @ C is always set here
	CALGN(	subcc	r2, r2, ip		)
	CALGN(	bcc	15f			)

11:		stmfd	sp!, {r5 - r9}
	UNWIND(	.fnend				)

	UNWIND(	.fnstart			)
	UNWIND(	.save	{r0, r4, lr}		)
	UNWIND(	.save	{r5 - r9}		) @ in new second stmfd block

	PLD(	pld	[r1, #-4]		)
	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #-32]		)
	PLD(	blt	13f			)
	PLD(	pld	[r1, #-64]		)
	PLD(	pld	[r1, #-96]		)

12:	PLD(	pld	[r1, #-128]		)
13:		ldmdb   r1!, {r7, r8, r9, ip}
		mov     lr, r3, lspush #\push
		subs    r2, r2, #32
		ldmdb   r1!, {r3, r4, r5, r6}
		orr     lr, lr, ip, lspull #\pull
		mov     ip, ip, lspush #\push
		orr     ip, ip, r9, lspull #\pull
		mov     r9, r9, lspush #\push
		orr     r9, r9, r8, lspull #\pull
		mov     r8, r8, lspush #\push
		orr     r8, r8, r7, lspull #\pull
		mov     r7, r7, lspush #\push
		orr     r7, r7, r6, lspull #\pull
		mov     r6, r6, lspush #\push
		orr     r6, r6, r5, lspull #\pull
		mov     r5, r5, lspush #\push
		orr     r5, r5, r4, lspull #\pull
		mov     r4, r4, lspush #\push
		orr     r4, r4, r3, lspull #\pull
		stmdb   r0!, {r4 - r9, ip, lr}
		bge	12b
	PLD(	cmn	r2, #96			)
	PLD(	bge	13b			)

		ldmfd	sp!, {r5 - r9}
	UNWIND(	.fnend				) @ end of the second stmfd block

	UNWIND(	.fnstart			)
	UNWIND(	.save {r0, r4, lr}		) @ still in first stmfd block

14:		ands	ip, r2, #28
		beq	16f

15:		mov     lr, r3, lspush #\push
		ldr	r3, [r1, #-4]!
		subs	ip, ip, #4
		orr	lr, lr, r3, lspull #\pull
		str	lr, [r0, #-4]!
		bgt	15b
	CALGN(	cmp	r2, #0			)
	CALGN(	bge	11b			)

16:		add	r1, r1, #(\pull / 8)
		b	8b
	UNWIND(	.fnend				)

		.endm


		backward_copy_shift	push=8	pull=24

17:		backward_copy_shift	push=16	pull=16

18:		backward_copy_shift	push=24	pull=8

ENDPROC(memmove)
EXPORT_SYMBOL(memmove)
Commit	Line	Data
75494230 NP	1	/*
	2	* linux/arch/arm/lib/memmove.S
	3	*
	4	* Author: Nicolas Pitre
	5	* Created: Sep 28, 2005
	6	* Copyright: (C) MontaVista Software Inc.
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License version 2 as
	10	* published by the Free Software Foundation.
	11	*/
	12
	13	#include <linux/linkage.h>
	14	#include <asm/assembler.h>
207a6cb0	15	#include <asm/unwind.h>
4dd1837d	16	#include <asm/export.h>
75494230	17
75494230 NP	18	.text
	19
	20	/*
	21	* Prototype: void memmove(void dest, const void *src, size_t n);
	22	*
	23	* Note:
	24	*
	25	* If the memory regions don't overlap, we simply branch to memcpy which is
	26	* normally a bit faster. Otherwise the copy is done going downwards. This
	27	* is a transposition of the code from copy_template.S but with the copy
	28	* occurring in the opposite direction.
	29	*/
	30
	31	ENTRY(memmove)
207a6cb0	32	UNWIND( .fnstart )
75494230 NP	33
	34	subs ip, r0, r1
	35	cmphi r2, ip
	36	bls memcpy
	37
	38	stmfd sp!, {r0, r4, lr}
207a6cb0 LY	39	UNWIND( .fnend )
	40
	41	UNWIND( .fnstart )
	42	UNWIND( .save {r0, r4, lr} ) @ in first stmfd block
75494230 NP	43	add r1, r1, r2
	44	add r0, r0, r2
	45	subs r2, r2, #4
	46	blt 8f
	47	ands ip, r0, #3
	48	PLD( pld [r1, #-4] )
	49	bne 9f
	50	ands ip, r1, #3
	51	bne 10f
	52
	53	1: subs r2, r2, #(28)
	54	stmfd sp!, {r5 - r8}
207a6cb0 LY	55	UNWIND( .fnend )
	56
	57	UNWIND( .fnstart )
	58	UNWIND( .save {r0, r4, lr} )
	59	UNWIND( .save {r5 - r8} ) @ in second stmfd block
75494230 NP	60	blt 5f
75494230 NP	61
2239aff6	62	CALGN( ands ip, r0, #31 )
75494230 NP	63	CALGN( sbcnes r4, ip, r2 ) @ C is always set here
	64	CALGN( bcs 2f )
	65	CALGN( adr r4, 6f )
	66	CALGN( subs r2, r2, ip ) @ C is set here
4c4925c1	67	CALGN( rsb ip, ip, #32 )
75494230 NP	68	CALGN( add pc, r4, ip )
	69
	70	PLD( pld [r1, #-4] )
	71	2: PLD( subs r2, r2, #96 )
	72	PLD( pld [r1, #-32] )
	73	PLD( blt 4f )
	74	PLD( pld [r1, #-64] )
	75	PLD( pld [r1, #-96] )
	76
	77	3: PLD( pld [r1, #-128] )
	78	4: ldmdb r1!, {r3, r4, r5, r6, r7, r8, ip, lr}
	79	subs r2, r2, #32
	80	stmdb r0!, {r3, r4, r5, r6, r7, r8, ip, lr}
	81	bge 3b
	82	PLD( cmn r2, #96 )
	83	PLD( bge 4b )
	84
	85	5: ands ip, r2, #28
	86	rsb ip, ip, #32
	87	addne pc, pc, ip @ C is always clear here
	88	b 7f
fd522a8d	89	6: W(nop)
8b592783 CM	90	W(ldr) r3, [r1, #-4]!
	91	W(ldr) r4, [r1, #-4]!
	92	W(ldr) r5, [r1, #-4]!
	93	W(ldr) r6, [r1, #-4]!
	94	W(ldr) r7, [r1, #-4]!
	95	W(ldr) r8, [r1, #-4]!
	96	W(ldr) lr, [r1, #-4]!
75494230 NP	97
	98	add pc, pc, ip
	99	nop
fd522a8d	100	W(nop)
8b592783 CM	101	W(str) r3, [r0, #-4]!
	102	W(str) r4, [r0, #-4]!
	103	W(str) r5, [r0, #-4]!
	104	W(str) r6, [r0, #-4]!
	105	W(str) r7, [r0, #-4]!
	106	W(str) r8, [r0, #-4]!
	107	W(str) lr, [r0, #-4]!
75494230 NP	108
	109	CALGN( bcs 2b )
	110
	111	7: ldmfd sp!, {r5 - r8}
207a6cb0 LY	112	UNWIND( .fnend ) @ end of second stmfd block
	113
	114	UNWIND( .fnstart )
	115	UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block
75494230 NP	116
	117	8: movs r2, r2, lsl #31
	118	ldrneb r3, [r1, #-1]!
	119	ldrcsb r4, [r1, #-1]!
	120	ldrcsb ip, [r1, #-1]
	121	strneb r3, [r0, #-1]!
	122	strcsb r4, [r0, #-1]!
	123	strcsb ip, [r0, #-1]
	124	ldmfd sp!, {r0, r4, pc}
	125
	126	9: cmp ip, #2
	127	ldrgtb r3, [r1, #-1]!
	128	ldrgeb r4, [r1, #-1]!
	129	ldrb lr, [r1, #-1]!
	130	strgtb r3, [r0, #-1]!
	131	strgeb r4, [r0, #-1]!
	132	subs r2, r2, ip
	133	strb lr, [r0, #-1]!
	134	blt 8b
	135	ands ip, r1, #3
	136	beq 1b
	137
	138	10: bic r1, r1, #3
	139	cmp ip, #2
	140	ldr r3, [r1, #0]
	141	beq 17f
	142	blt 18f
207a6cb0	143	UNWIND( .fnend )
75494230 NP	144
	145
	146	.macro backward_copy_shift push pull
	147
207a6cb0 LY	148	UNWIND( .fnstart )
207a6cb0 LY	149	UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block
75494230 NP	150	subs r2, r2, #28
	151	blt 14f
	152
2239aff6	153	CALGN( ands ip, r0, #31 )
75494230 NP	154	CALGN( sbcnes r4, ip, r2 ) @ C is always set here
	155	CALGN( subcc r2, r2, ip )
	156	CALGN( bcc 15f )
	157
	158	11: stmfd sp!, {r5 - r9}
207a6cb0 LY	159	UNWIND( .fnend )
	160
	161	UNWIND( .fnstart )
	162	UNWIND( .save {r0, r4, lr} )
	163	UNWIND( .save {r5 - r9} ) @ in new second stmfd block
75494230 NP	164
	165	PLD( pld [r1, #-4] )
	166	PLD( subs r2, r2, #96 )
	167	PLD( pld [r1, #-32] )
	168	PLD( blt 13f )
	169	PLD( pld [r1, #-64] )
	170	PLD( pld [r1, #-96] )
	171
	172	12: PLD( pld [r1, #-128] )
	173	13: ldmdb r1!, {r7, r8, r9, ip}
d98b90ea	174	mov lr, r3, lspush #\push
75494230 NP	175	subs r2, r2, #32
75494230 NP	176	ldmdb r1!, {r3, r4, r5, r6}
d98b90ea VK	177	orr lr, lr, ip, lspull #\pull
	178	mov ip, ip, lspush #\push
	179	orr ip, ip, r9, lspull #\pull
	180	mov r9, r9, lspush #\push
	181	orr r9, r9, r8, lspull #\pull
	182	mov r8, r8, lspush #\push
	183	orr r8, r8, r7, lspull #\pull
	184	mov r7, r7, lspush #\push
	185	orr r7, r7, r6, lspull #\pull
	186	mov r6, r6, lspush #\push
	187	orr r6, r6, r5, lspull #\pull
	188	mov r5, r5, lspush #\push
	189	orr r5, r5, r4, lspull #\pull
	190	mov r4, r4, lspush #\push
	191	orr r4, r4, r3, lspull #\pull
75494230 NP	192	stmdb r0!, {r4 - r9, ip, lr}
	193	bge 12b
	194	PLD( cmn r2, #96 )
	195	PLD( bge 13b )
	196
	197	ldmfd sp!, {r5 - r9}
207a6cb0 LY	198	UNWIND( .fnend ) @ end of the second stmfd block
	199
	200	UNWIND( .fnstart )
	201	UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block
75494230 NP	202
	203	14: ands ip, r2, #28
	204	beq 16f
	205
d98b90ea	206	15: mov lr, r3, lspush #\push
75494230 NP	207	ldr r3, [r1, #-4]!
75494230 NP	208	subs ip, ip, #4
d98b90ea	209	orr lr, lr, r3, lspull #\pull
75494230 NP	210	str lr, [r0, #-4]!
	211	bgt 15b
	212	CALGN( cmp r2, #0 )
	213	CALGN( bge 11b )
	214
	215	16: add r1, r1, #(\pull / 8)
	216	b 8b
207a6cb0	217	UNWIND( .fnend )
75494230 NP	218
	219	.endm
	220
	221
	222	backward_copy_shift push=8 pull=24
	223
	224	17: backward_copy_shift push=16 pull=16
	225
	226	18: backward_copy_shift push=24 pull=8
	227
93ed3970	228	ENDPROC(memmove)
4dd1837d	229	EXPORT_SYMBOL(memmove)