[deliverable/linux.git] / arch / powerpc / lib / memcpy_64.S

/*
 * Copyright (C) 2002 Paul Mackerras, IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */
#include <asm/processor.h>
#include <asm/ppc_asm.h>

	.align	7
_GLOBAL_TOC(memcpy)
BEGIN_FTR_SECTION
#ifdef __LITTLE_ENDIAN__
	cmpdi	cr7,r5,0
#else
	std	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* save destination pointer for return value */
#endif
FTR_SECTION_ELSE
#ifndef SELFTEST
	b	memcpy_power7
#endif
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
#ifdef __LITTLE_ENDIAN__
	/* dumb little-endian memcpy that will get replaced at runtime */
	addi r9,r3,-1
	addi r4,r4,-1
	beqlr cr7
	mtctr r5
1:	lbzu r10,1(r4)
	stbu r10,1(r9)
	bdnz 1b
	blr
#else
	PPC_MTOCRF(0x01,r5)
	cmpldi	cr1,r5,16
	neg	r6,r3		# LS 3 bits = # bytes to 8-byte dest bdry
	andi.	r6,r6,7
	dcbt	0,r4
	blt	cr1,.Lshort_copy
/* Below we want to nop out the bne if we're on a CPU that has the
   CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
   cleared.
   At the time of writing the only CPU that has this combination of bits
   set is Power6. */
BEGIN_FTR_SECTION
	nop
FTR_SECTION_ELSE
	bne	.Ldst_unaligned
ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \
                    CPU_FTR_UNALIGNED_LD_STD)
.Ldst_aligned:
	addi	r3,r3,-16
BEGIN_FTR_SECTION
	andi.	r0,r4,7
	bne	.Lsrc_unaligned
END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
	srdi	r7,r5,4
	ld	r9,0(r4)
	addi	r4,r4,-8
	mtctr	r7
	andi.	r5,r5,7
	bf	cr7*4+0,2f
	addi	r3,r3,8
	addi	r4,r4,8
	mr	r8,r9
	blt	cr1,3f
1:	ld	r9,8(r4)
	std	r8,8(r3)
2:	ldu	r8,16(r4)
	stdu	r9,16(r3)
	bdnz	1b
3:	std	r8,8(r3)
	beq	3f
	addi	r3,r3,16
.Ldo_tail:
	bf	cr7*4+1,1f
	lwz	r9,8(r4)
	addi	r4,r4,4
	stw	r9,0(r3)
	addi	r3,r3,4
1:	bf	cr7*4+2,2f
	lhz	r9,8(r4)
	addi	r4,r4,2
	sth	r9,0(r3)
	addi	r3,r3,2
2:	bf	cr7*4+3,3f
	lbz	r9,8(r4)
	stb	r9,0(r3)
3:	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* return dest pointer */
	blr

.Lsrc_unaligned:
	srdi	r6,r5,3
	addi	r5,r5,-16
	subf	r4,r0,r4
	srdi	r7,r5,4
	sldi	r10,r0,3
	cmpdi	cr6,r6,3
	andi.	r5,r5,7
	mtctr	r7
	subfic	r11,r10,64
	add	r5,r5,r0

	bt	cr7*4+0,0f

	ld	r9,0(r4)	# 3+2n loads, 2+2n stores
	ld	r0,8(r4)
	sld	r6,r9,r10
	ldu	r9,16(r4)
	srd	r7,r0,r11
	sld	r8,r0,r10
	or	r7,r7,r6
	blt	cr6,4f
	ld	r0,8(r4)
	# s1<< in r8, d0=(s0<<|s1>>) in r7, s3 in r0, s2 in r9, nix in r6 & r12
	b	2f

0:	ld	r0,0(r4)	# 4+2n loads, 3+2n stores
	ldu	r9,8(r4)
	sld	r8,r0,r10
	addi	r3,r3,-8
	blt	cr6,5f
	ld	r0,8(r4)
	srd	r12,r9,r11
	sld	r6,r9,r10
	ldu	r9,16(r4)
	or	r12,r8,r12
	srd	r7,r0,r11
	sld	r8,r0,r10
	addi	r3,r3,16
	beq	cr6,3f

	# d0=(s0<<|s1>>) in r12, s1<< in r6, s2>> in r7, s2<< in r8, s3 in r9
1:	or	r7,r7,r6
	ld	r0,8(r4)
	std	r12,8(r3)
2:	srd	r12,r9,r11
	sld	r6,r9,r10
	ldu	r9,16(r4)
	or	r12,r8,r12
	stdu	r7,16(r3)
	srd	r7,r0,r11
	sld	r8,r0,r10
	bdnz	1b

3:	std	r12,8(r3)
	or	r7,r7,r6
4:	std	r7,16(r3)
5:	srd	r12,r9,r11
	or	r12,r8,r12
	std	r12,24(r3)
	beq	4f
	cmpwi	cr1,r5,8
	addi	r3,r3,32
	sld	r9,r9,r10
	ble	cr1,6f
	ld	r0,8(r4)
	srd	r7,r0,r11
	or	r9,r7,r9
6:
	bf	cr7*4+1,1f
	rotldi	r9,r9,32
	stw	r9,0(r3)
	addi	r3,r3,4
1:	bf	cr7*4+2,2f
	rotldi	r9,r9,16
	sth	r9,0(r3)
	addi	r3,r3,2
2:	bf	cr7*4+3,3f
	rotldi	r9,r9,8
	stb	r9,0(r3)
3:	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* return dest pointer */
	blr

.Ldst_unaligned:
	PPC_MTOCRF(0x01,r6)		# put #bytes to 8B bdry into cr7
	subf	r5,r6,r5
	li	r7,0
	cmpldi	cr1,r5,16
	bf	cr7*4+3,1f
	lbz	r0,0(r4)
	stb	r0,0(r3)
	addi	r7,r7,1
1:	bf	cr7*4+2,2f
	lhzx	r0,r7,r4
	sthx	r0,r7,r3
	addi	r7,r7,2
2:	bf	cr7*4+1,3f
	lwzx	r0,r7,r4
	stwx	r0,r7,r3
3:	PPC_MTOCRF(0x01,r5)
	add	r4,r6,r4
	add	r3,r6,r3
	b	.Ldst_aligned

.Lshort_copy:
	bf	cr7*4+0,1f
	lwz	r0,0(r4)
	lwz	r9,4(r4)
	addi	r4,r4,8
	stw	r0,0(r3)
	stw	r9,4(r3)
	addi	r3,r3,8
1:	bf	cr7*4+1,2f
	lwz	r0,0(r4)
	addi	r4,r4,4
	stw	r0,0(r3)
	addi	r3,r3,4
2:	bf	cr7*4+2,3f
	lhz	r0,0(r4)
	addi	r4,r4,2
	sth	r0,0(r3)
	addi	r3,r3,2
3:	bf	cr7*4+3,4f
	lbz	r0,0(r4)
	stb	r0,0(r3)
4:	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* return dest pointer */
	blr
#endif
Commit	Line	Data
14cf11af	1	/*
14cf11af PM	2	* Copyright (C) 2002 Paul Mackerras, IBM Corp.
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation; either version
	7	* 2 of the License, or (at your option) any later version.
	8	*/
	9	#include <asm/processor.h>
	10	#include <asm/ppc_asm.h>
	11
	12	.align 7
169c7cee	13	_GLOBAL_TOC(memcpy)
b3f271e8	14	BEGIN_FTR_SECTION
00f554fa PB	15	#ifdef __LITTLE_ENDIAN__
	16	cmpdi cr7,r5,0
	17	#else
752a6422	18	std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* save destination pointer for return value */
00f554fa	19	#endif
b3f271e8	20	FTR_SECTION_ELSE
22d651dc	21	#ifndef SELFTEST
b3f271e8	22	b memcpy_power7
22d651dc	23	#endif
b3f271e8	24	ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
00f554fa PB	25	#ifdef __LITTLE_ENDIAN__
	26	/* dumb little-endian memcpy that will get replaced at runtime */
	27	addi r9,r3,-1
	28	addi r4,r4,-1
	29	beqlr cr7
	30	mtctr r5
	31	1: lbzu r10,1(r4)
	32	stbu r10,1(r9)
	33	bdnz 1b
	34	blr
	35	#else
86e32fdc	36	PPC_MTOCRF(0x01,r5)
14cf11af PM	37	cmpldi cr1,r5,16
	38	neg r6,r3 # LS 3 bits = # bytes to 8-byte dest bdry
	39	andi. r6,r6,7
	40	dcbt 0,r4
	41	blt cr1,.Lshort_copy
25d6e2d7 MN	42	/* Below we want to nop out the bne if we're on a CPU that has the
	43	CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
	44	cleared.
	45	At the time of writing the only CPU that has this combination of bits
	46	set is Power6. */
	47	BEGIN_FTR_SECTION
	48	nop
	49	FTR_SECTION_ELSE
14cf11af	50	bne .Ldst_unaligned
25d6e2d7 MN	51	ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD \| CPU_FTR_CP_USE_DCBTZ, \
25d6e2d7 MN	52	CPU_FTR_UNALIGNED_LD_STD)
14cf11af	53	.Ldst_aligned:
14cf11af	54	addi r3,r3,-16
25d6e2d7 MN	55	BEGIN_FTR_SECTION
25d6e2d7 MN	56	andi. r0,r4,7
14cf11af	57	bne .Lsrc_unaligned
25d6e2d7	58	END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
14cf11af PM	59	srdi r7,r5,4
	60	ld r9,0(r4)
	61	addi r4,r4,-8
	62	mtctr r7
	63	andi. r5,r5,7
	64	bf cr7*4+0,2f
	65	addi r3,r3,8
	66	addi r4,r4,8
	67	mr r8,r9
	68	blt cr1,3f
	69	1: ld r9,8(r4)
	70	std r8,8(r3)
	71	2: ldu r8,16(r4)
	72	stdu r9,16(r3)
	73	bdnz 1b
	74	3: std r8,8(r3)
d0027bf0	75	beq 3f
14cf11af	76	addi r3,r3,16
14cf11af PM	77	.Ldo_tail:
14cf11af PM	78	bf cr7*4+1,1f
e423b9ec MN	79	lwz r9,8(r4)
e423b9ec MN	80	addi r4,r4,4
14cf11af PM	81	stw r9,0(r3)
	82	addi r3,r3,4
	83	1: bf cr7*4+2,2f
e423b9ec MN	84	lhz r9,8(r4)
e423b9ec MN	85	addi r4,r4,2
14cf11af PM	86	sth r9,0(r3)
	87	addi r3,r3,2
	88	2: bf cr7*4+3,3f
e423b9ec	89	lbz r9,8(r4)
14cf11af	90	stb r9,0(r3)
752a6422	91	3: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */
d0027bf0	92	blr
14cf11af PM	93
	94	.Lsrc_unaligned:
	95	srdi r6,r5,3
	96	addi r5,r5,-16
	97	subf r4,r0,r4
	98	srdi r7,r5,4
	99	sldi r10,r0,3
	100	cmpdi cr6,r6,3
	101	andi. r5,r5,7
	102	mtctr r7
	103	subfic r11,r10,64
	104	add r5,r5,r0
	105
	106	bt cr7*4+0,0f
	107
	108	ld r9,0(r4) # 3+2n loads, 2+2n stores
	109	ld r0,8(r4)
	110	sld r6,r9,r10
	111	ldu r9,16(r4)
	112	srd r7,r0,r11
	113	sld r8,r0,r10
	114	or r7,r7,r6
	115	blt cr6,4f
	116	ld r0,8(r4)
	117	# s1<< in r8, d0=(s0<<\|s1>>) in r7, s3 in r0, s2 in r9, nix in r6 & r12
	118	b 2f
	119
	120	0: ld r0,0(r4) # 4+2n loads, 3+2n stores
	121	ldu r9,8(r4)
	122	sld r8,r0,r10
	123	addi r3,r3,-8
	124	blt cr6,5f
	125	ld r0,8(r4)
	126	srd r12,r9,r11
	127	sld r6,r9,r10
	128	ldu r9,16(r4)
	129	or r12,r8,r12
	130	srd r7,r0,r11
	131	sld r8,r0,r10
	132	addi r3,r3,16
	133	beq cr6,3f
	134
	135	# d0=(s0<<\|s1>>) in r12, s1<< in r6, s2>> in r7, s2<< in r8, s3 in r9
	136	1: or r7,r7,r6
	137	ld r0,8(r4)
	138	std r12,8(r3)
	139	2: srd r12,r9,r11
	140	sld r6,r9,r10
	141	ldu r9,16(r4)
	142	or r12,r8,r12
	143	stdu r7,16(r3)
	144	srd r7,r0,r11
	145	sld r8,r0,r10
	146	bdnz 1b
	147
	148	3: std r12,8(r3)
	149	or r7,r7,r6
	150	4: std r7,16(r3)
	151	5: srd r12,r9,r11
	152	or r12,r8,r12
	153	std r12,24(r3)
d0027bf0	154	beq 4f
14cf11af PM	155	cmpwi cr1,r5,8
	156	addi r3,r3,32
	157	sld r9,r9,r10
e423b9ec	158	ble cr1,6f
14cf11af PM	159	ld r0,8(r4)
	160	srd r7,r0,r11
	161	or r9,r7,r9
e423b9ec MN	162	6:
	163	bf cr7*4+1,1f
	164	rotldi r9,r9,32
	165	stw r9,0(r3)
	166	addi r3,r3,4
	167	1: bf cr7*4+2,2f
	168	rotldi r9,r9,16
	169	sth r9,0(r3)
	170	addi r3,r3,2
	171	2: bf cr7*4+3,3f
	172	rotldi r9,r9,8
	173	stb r9,0(r3)
752a6422	174	3: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */
e423b9ec	175	blr
14cf11af PM	176
14cf11af PM	177	.Ldst_unaligned:
86e32fdc	178	PPC_MTOCRF(0x01,r6) # put #bytes to 8B bdry into cr7
14cf11af PM	179	subf r5,r6,r5
14cf11af PM	180	li r7,0
25d6e2d7	181	cmpldi cr1,r5,16
14cf11af PM	182	bf cr7*4+3,1f
	183	lbz r0,0(r4)
	184	stb r0,0(r3)
	185	addi r7,r7,1
	186	1: bf cr7*4+2,2f
	187	lhzx r0,r7,r4
	188	sthx r0,r7,r3
	189	addi r7,r7,2
	190	2: bf cr7*4+1,3f
	191	lwzx r0,r7,r4
	192	stwx r0,r7,r3
86e32fdc	193	3: PPC_MTOCRF(0x01,r5)
14cf11af PM	194	add r4,r6,r4
	195	add r3,r6,r3
	196	b .Ldst_aligned
	197
	198	.Lshort_copy:
	199	bf cr7*4+0,1f
	200	lwz r0,0(r4)
	201	lwz r9,4(r4)
	202	addi r4,r4,8
	203	stw r0,0(r3)
	204	stw r9,4(r3)
	205	addi r3,r3,8
	206	1: bf cr7*4+1,2f
	207	lwz r0,0(r4)
	208	addi r4,r4,4
	209	stw r0,0(r3)
	210	addi r3,r3,4
	211	2: bf cr7*4+2,3f
	212	lhz r0,0(r4)
	213	addi r4,r4,2
	214	sth r0,0(r3)
	215	addi r3,r3,2
	216	3: bf cr7*4+3,4f
	217	lbz r0,0(r4)
	218	stb r0,0(r3)
752a6422	219	4: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */
d0027bf0	220	blr
00f554fa	221	#endif