[deliverable/linux.git] / tools / testing / selftests / rseq / rseq-ppc.h

/*
 * rseq-ppc.h
 *
 * (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * (C) Copyright 2016 - Boqun Feng <boqun.feng@gmail.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#define smp_mb()	__asm__ __volatile__ ("sync" : : : "memory")
#define smp_lwsync()	__asm__ __volatile__ ("lwsync" : : : "memory")
#define smp_rmb()	smp_lwsync()
#define smp_wmb()	smp_lwsync()

#define smp_load_acquire(p)						\
__extension__ ({							\
	__typeof(*p) ____p1 = READ_ONCE(*p);				\
	smp_lwsync();							\
	____p1;								\
})

#define smp_acquire__after_ctrl_dep()	smp_lwsync()

#define smp_store_release(p, v)						\
do {									\
	smp_lwsync();							\
	WRITE_ONCE(*p, v);						\
} while (0)

#define has_fast_acquire_release()	0

#ifdef __PPC64__
#define has_single_copy_load_64()	1
#else
#define has_single_copy_load_64()	0
#endif

/*
 * The __rseq_table section can be used by debuggers to better handle
 * single-stepping through the restartable critical sections.
 */

#ifdef __PPC64__

#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
		_failure, _spec_store, _spec_input, \
		_final_store, _final_input, _extra_clobber, \
		_setup, _teardown, _scratch) \
	__asm__ __volatile__ goto ( \
		".pushsection __rseq_table, \"aw\"\n\t" \
		".balign 32\n\t" \
		"3:\n\t" \
		".quad 1f, 2f, 4f, 0x0\n\t" \
		".popsection\n\t" \
		"1:\n\t" \
		_setup \
		RSEQ_INJECT_ASM(1) \
		"lis %%r17, (3b)@highest\n\t" \
		"ori %%r17, %%r17, (3b)@higher\n\t" \
		"rldicr %%r17, %%r17, 32, 31\n\t" \
		"oris %%r17, %%r17, (3b)@h\n\t" \
		"ori %%r17, %%r17, (3b)@l\n\t" \
		"std %%r17, 0(%[rseq_cs])\n\t" \
		RSEQ_INJECT_ASM(2) \
		"lwz %%r17, %[current_event_counter]\n\t" \
		"cmpw cr7, %[start_event_counter], %%r17\n\t" \
		"bne- cr7, 4f\n\t" \
		RSEQ_INJECT_ASM(3) \
		_spec_store \
		_final_store \
		"2:\n\t" \
		RSEQ_INJECT_ASM(5) \
		"li %%r17, 0\n\t" \
		"std %%r17, 0(%[rseq_cs])\n\t" \
		_teardown \
		"b 5f\n\t" \
		"4:\n\t" \
		"li %%r17, 0\n\t" \
		"std %%r17, 0(%[rseq_cs])\n\t" \
		_teardown \
		"b %l[failure]\n\t" \
		"5:\n\t" \
		: /* gcc asm goto does not allow outputs */ \
		: [start_event_counter]"r"((_start_value).event_counter), \
		  [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
		  [rseq_cs]"b"(&(_start_value).rseqp->rseq_cs) \
		  _spec_input \
		  _final_input \
		  RSEQ_INJECT_INPUT \
		: "r17", "memory", "cc" \
		  _extra_clobber \
		  RSEQ_INJECT_CLOBBER \
		: _failure \
	)

#define RSEQ_FINISH_FINAL_STORE_ASM() \
		"std %[to_write_final], 0(%[target_final])\n\t"

#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
		"lwsync\n\t" \
		RSEQ_FINISH_FINAL_STORE_ASM()

#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
		, [to_write_final]"r"(_to_write_final), \
		[target_final]"b"(_target_final)

#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
		"std %[to_write_spec], 0(%[target_spec])\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
		, [to_write_spec]"r"(_to_write_spec), \
		[target_spec]"b"(_target_spec)

/* TODO: implement a faster memcpy. */
#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
		"cmpdi %%r19, 0\n\t" \
		"beq 333f\n\t" \
		"addi %%r20, %%r20, -1\n\t" \
		"addi %%r21, %%r21, -1\n\t" \
		"222:\n\t" \
		"lbzu %%r18, 1(%%r20)\n\t" \
		"stbu %%r18, 1(%%r21)\n\t" \
		"addi %%r19, %%r19, -1\n\t" \
		"cmpdi %%r19, 0\n\t" \
		"bne 222b\n\t" \
		"333:\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
		, [to_write_memcpy]"r"(_to_write_memcpy), \
		[target_memcpy]"r"(_target_memcpy), \
		[len_memcpy]"r"(_len_memcpy)

#define RSEQ_FINISH_MEMCPY_CLOBBER() \
		, "r18", "r19", "r20", "r21"

#define RSEQ_FINISH_MEMCPY_SCRATCH()

/*
 * We use extra registers to hold the input registers, and we don't need to
 * save and restore the input registers.
 */
#define RSEQ_FINISH_MEMCPY_SETUP() \
		"mr %%r19, %[len_memcpy]\n\t" \
		"mr %%r20, %[to_write_memcpy]\n\t" \
		"mr %%r21, %[target_memcpy]\n\t" \

#define RSEQ_FINISH_MEMCPY_TEARDOWN()

#else	/* #ifdef __PPC64__ */

#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
		_failure, _spec_store, _spec_input, \
		_final_store, _final_input, _extra_clobber, \
		_setup, _teardown, _scratch) \
	__asm__ __volatile__ goto ( \
		".pushsection __rseq_table, \"aw\"\n\t" \
		".balign 32\n\t" \
		"3:\n\t" \
		/* 32-bit only supported on BE */ \
		".long 0x0, 1f, 0x0, 2f, 0x0, 4f, 0x0, 0x0\n\t" \
		".popsection\n\t" \
		"1:\n\t" \
		_setup \
		RSEQ_INJECT_ASM(1) \
		"lis %%r17, (3b)@ha\n\t" \
		"addi %%r17, %%r17, (3b)@l\n\t" \
		"stw %%r17, 0(%[rseq_cs])\n\t" \
		RSEQ_INJECT_ASM(2) \
		"lwz %%r17, %[current_event_counter]\n\t" \
		"cmpw cr7, %[start_event_counter], %%r17\n\t" \
		"bne- cr7, 4f\n\t" \
		RSEQ_INJECT_ASM(3) \
		_spec_store \
		_final_store \
		"2:\n\t" \
		RSEQ_INJECT_ASM(5) \
		"li %%r17, 0\n\t" \
		"stw %%r17, 0(%[rseq_cs])\n\t" \
		_teardown \
		"b 5f\n\t" \
		"4:\n\t" \
		"li %%r17, 0\n\t" \
		"stw %%r17, 0(%[rseq_cs])\n\t" \
		_teardown \
		"b %l[failure]\n\t" \
		"5:\n\t" \
		: /* gcc asm goto does not allow outputs */ \
		: [start_event_counter]"r"((_start_value).event_counter), \
		  [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
		  [rseq_cs]"b"(&(_start_value).rseqp->rseq_cs) \
		  _spec_input \
		  _final_input \
		  RSEQ_INJECT_INPUT \
		: "r17", "memory", "cc" \
		  _extra_clobber \
		  RSEQ_INJECT_CLOBBER \
		: _failure \
	)

#define RSEQ_FINISH_FINAL_STORE_ASM() \
		"stw %[to_write_final], 0(%[target_final])\n\t"

#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
		"lwsync\n\t" \
		RSEQ_FINISH_FINAL_STORE_ASM()

#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
		, [to_write_final]"r"(_to_write_final), \
		[target_final]"b"(_target_final)

#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
		"stw %[to_write_spec], 0(%[target_spec])\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
		, [to_write_spec]"r"(_to_write_spec), \
		[target_spec]"b"(_target_spec)

/* TODO: implement a faster memcpy. */
#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
		"cmpwi %%r19, 0\n\t" \
		"beq 333f\n\t" \
		"addi %%r20, %%r20, -1\n\t" \
		"addi %%r21, %%r21, -1\n\t" \
		"222:\n\t" \
		"lbzu %%r18, 1(%%r20)\n\t" \
		"stbu %%r18, 1(%%r21)\n\t" \
		"addi %%r19, %%r19, -1\n\t" \
		"cmpwi %%r19, 0\n\t" \
		"bne 222b\n\t" \
		"333:\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
		, [to_write_memcpy]"r"(_to_write_memcpy), \
		[target_memcpy]"r"(_target_memcpy), \
		[len_memcpy]"r"(_len_memcpy)

#define RSEQ_FINISH_MEMCPY_CLOBBER() \
		, "r18", "r19", "r20", "r21"

#define RSEQ_FINISH_MEMCPY_SCRATCH()

/*
 * We use extra registers to hold the input registers, and we don't need to
 * save and restore the input registers.
 */
#define RSEQ_FINISH_MEMCPY_SETUP() \
		"mr %%r19, %[len_memcpy]\n\t" \
		"mr %%r20, %[to_write_memcpy]\n\t" \
		"mr %%r21, %[target_memcpy]\n\t" \

#define RSEQ_FINISH_MEMCPY_TEARDOWN()

#endif	/* #else #ifdef __PPC64__ */
Commit	Line	Data
b54c5158 MD	1	/*
	2	* rseq-ppc.h
	3	*
	4	* (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	5	* (C) Copyright 2016 - Boqun Feng <boqun.feng@gmail.com>
	6	*
	7	* Permission is hereby granted, free of charge, to any person obtaining a copy
	8	* of this software and associated documentation files (the "Software"), to deal
	9	* in the Software without restriction, including without limitation the rights
	10	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	11	* copies of the Software, and to permit persons to whom the Software is
	12	* furnished to do so, subject to the following conditions:
	13	*
	14	* The above copyright notice and this permission notice shall be included in
	15	* all copies or substantial portions of the Software.
	16	*
	17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	18	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	19	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	20	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	21	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	22	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	23	* SOFTWARE.
	24	*/
	25
	26	#define smp_mb() __asm__ __volatile__ ("sync" : : : "memory")
	27	#define smp_lwsync() __asm__ __volatile__ ("lwsync" : : : "memory")
	28	#define smp_rmb() smp_lwsync()
	29	#define smp_wmb() smp_lwsync()
	30
	31	#define smp_load_acquire(p) \
	32	__extension__ ({ \
	33	__typeof(p) ____p1 = READ_ONCE(p); \
	34	smp_lwsync(); \
	35	____p1; \
	36	})
	37
	38	#define smp_acquire__after_ctrl_dep() smp_lwsync()
	39
	40	#define smp_store_release(p, v) \
	41	do { \
	42	smp_lwsync(); \
	43	WRITE_ONCE(*p, v); \
	44	} while (0)
	45
	46	#define has_fast_acquire_release() 0
	47
	48	#ifdef __PPC64__
	49	#define has_single_copy_load_64() 1
	50	#else
	51	#define has_single_copy_load_64() 0
	52	#endif
	53
	54	/*
	55	* The __rseq_table section can be used by debuggers to better handle
	56	* single-stepping through the restartable critical sections.
	57	*/
	58
	59	#ifdef __PPC64__
	60
	61	#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
	62	_failure, _spec_store, _spec_input, \
	63	_final_store, _final_input, _extra_clobber, \
	64	_setup, _teardown, _scratch) \
65	__asm__ __volatile__ goto ( \
66	".pushsection __rseq_table, \"aw\"\n\t" \
67	".balign 32\n\t" \
68	"3:\n\t" \
69	".quad 1f, 2f, 4f, 0x0\n\t" \
70	".popsection\n\t" \
71	"1:\n\t" \
72	_setup \
73	RSEQ_INJECT_ASM(1) \
74	"lis %%r17, (3b)@highest\n\t" \
75	"ori %%r17, %%r17, (3b)@higher\n\t" \
76	"rldicr %%r17, %%r17, 32, 31\n\t" \
77	"oris %%r17, %%r17, (3b)@h\n\t" \
78	"ori %%r17, %%r17, (3b)@l\n\t" \
79	"std %%r17, 0(%[rseq_cs])\n\t" \
80	RSEQ_INJECT_ASM(2) \
81	"lwz %%r17, %[current_event_counter]\n\t" \
82	"cmpw cr7, %[start_event_counter], %%r17\n\t" \
83	"bne- cr7, 4f\n\t" \
84	RSEQ_INJECT_ASM(3) \
85	_spec_store \
86	_final_store \
87	"2:\n\t" \
88	RSEQ_INJECT_ASM(5) \
89	"li %%r17, 0\n\t" \
90	"std %%r17, 0(%[rseq_cs])\n\t" \
91	_teardown \
92	"b 5f\n\t" \
93	"4:\n\t" \
94	"li %%r17, 0\n\t" \
95	"std %%r17, 0(%[rseq_cs])\n\t" \
96	_teardown \
97	"b %l[failure]\n\t" \
98	"5:\n\t" \
99	: /* gcc asm goto does not allow outputs */ \
100	: [start_event_counter]"r"((_start_value).event_counter), \
101	[current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
102	[rseq_cs]"b"(&(_start_value).rseqp->rseq_cs) \
103	_spec_input \
104	_final_input \
105	RSEQ_INJECT_INPUT \
106	: "r17", "memory", "cc" \
107	_extra_clobber \
108	RSEQ_INJECT_CLOBBER \
109	: _failure \
110	)
111
112	#define RSEQ_FINISH_FINAL_STORE_ASM() \
113	"std %[to_write_final], 0(%[target_final])\n\t"
114
115	#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
116	"lwsync\n\t" \
117	RSEQ_FINISH_FINAL_STORE_ASM()
118
119	#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
120	, [to_write_final]"r"(_to_write_final), \
121	[target_final]"b"(_target_final)
122
123	#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
124	"std %[to_write_spec], 0(%[target_spec])\n\t" \
125	RSEQ_INJECT_ASM(4)
126
127	#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
128	, [to_write_spec]"r"(_to_write_spec), \
129	[target_spec]"b"(_target_spec)
130
131	/* TODO: implement a faster memcpy. */
132	#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
133	"cmpdi %%r19, 0\n\t" \
134	"beq 333f\n\t" \
135	"addi %%r20, %%r20, -1\n\t" \
136	"addi %%r21, %%r21, -1\n\t" \
137	"222:\n\t" \
138	"lbzu %%r18, 1(%%r20)\n\t" \
139	"stbu %%r18, 1(%%r21)\n\t" \
140	"addi %%r19, %%r19, -1\n\t" \
141	"cmpdi %%r19, 0\n\t" \
142	"bne 222b\n\t" \
143	"333:\n\t" \
144	RSEQ_INJECT_ASM(4)
145
146	#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
147	, [to_write_memcpy]"r"(_to_write_memcpy), \
148	[target_memcpy]"r"(_target_memcpy), \
149	[len_memcpy]"r"(_len_memcpy)
150
151	#define RSEQ_FINISH_MEMCPY_CLOBBER() \
152	, "r18", "r19", "r20", "r21"
153
154	#define RSEQ_FINISH_MEMCPY_SCRATCH()
155
156	/*
157	* We use extra registers to hold the input registers, and we don't need to
158	* save and restore the input registers.
159	*/
160	#define RSEQ_FINISH_MEMCPY_SETUP() \
161	"mr %%r19, %[len_memcpy]\n\t" \
162	"mr %%r20, %[to_write_memcpy]\n\t" \
163	"mr %%r21, %[target_memcpy]\n\t" \
164
165	#define RSEQ_FINISH_MEMCPY_TEARDOWN()
166
167	#else /* #ifdef __PPC64__ */
168
169	#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
170	_failure, _spec_store, _spec_input, \
171	_final_store, _final_input, _extra_clobber, \
172	_setup, _teardown, _scratch) \
173	__asm__ __volatile__ goto ( \
174	".pushsection __rseq_table, \"aw\"\n\t" \
175	".balign 32\n\t" \
176	"3:\n\t" \
177	/* 32-bit only supported on BE */ \
178	".long 0x0, 1f, 0x0, 2f, 0x0, 4f, 0x0, 0x0\n\t" \
179	".popsection\n\t" \
180	"1:\n\t" \
181	_setup \
182	RSEQ_INJECT_ASM(1) \
183	"lis %%r17, (3b)@ha\n\t" \
184	"addi %%r17, %%r17, (3b)@l\n\t" \
185	"stw %%r17, 0(%[rseq_cs])\n\t" \
186	RSEQ_INJECT_ASM(2) \
187	"lwz %%r17, %[current_event_counter]\n\t" \
188	"cmpw cr7, %[start_event_counter], %%r17\n\t" \
189	"bne- cr7, 4f\n\t" \
190	RSEQ_INJECT_ASM(3) \
191	_spec_store \
192	_final_store \
193	"2:\n\t" \
194	RSEQ_INJECT_ASM(5) \
195	"li %%r17, 0\n\t" \
196	"stw %%r17, 0(%[rseq_cs])\n\t" \
197	_teardown \
198	"b 5f\n\t" \
199	"4:\n\t" \
200	"li %%r17, 0\n\t" \
820319e6	201	"stw %%r17, 0(%[rseq_cs])\n\t" \
b54c5158 MD	202	_teardown \
	203	"b %l[failure]\n\t" \
	204	"5:\n\t" \
	205	: /* gcc asm goto does not allow outputs */ \
	206	: [start_event_counter]"r"((_start_value).event_counter), \
	207	[current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
	208	[rseq_cs]"b"(&(_start_value).rseqp->rseq_cs) \
	209	_spec_input \
	210	_final_input \
	211	RSEQ_INJECT_INPUT \
	212	: "r17", "memory", "cc" \
	213	_extra_clobber \
	214	RSEQ_INJECT_CLOBBER \
	215	: _failure \
	216	)
	217
	218	#define RSEQ_FINISH_FINAL_STORE_ASM() \
	219	"stw %[to_write_final], 0(%[target_final])\n\t"
	220
	221	#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
	222	"lwsync\n\t" \
	223	RSEQ_FINISH_FINAL_STORE_ASM()
	224
	225	#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
	226	, [to_write_final]"r"(_to_write_final), \
	227	[target_final]"b"(_target_final)
	228
	229	#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
	230	"stw %[to_write_spec], 0(%[target_spec])\n\t" \
	231	RSEQ_INJECT_ASM(4)
	232
	233	#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
	234	, [to_write_spec]"r"(_to_write_spec), \
	235	[target_spec]"b"(_target_spec)
	236
	237	/* TODO: implement a faster memcpy. */
	238	#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
	239	"cmpwi %%r19, 0\n\t" \
	240	"beq 333f\n\t" \
	241	"addi %%r20, %%r20, -1\n\t" \
	242	"addi %%r21, %%r21, -1\n\t" \
	243	"222:\n\t" \
	244	"lbzu %%r18, 1(%%r20)\n\t" \
	245	"stbu %%r18, 1(%%r21)\n\t" \
	246	"addi %%r19, %%r19, -1\n\t" \
	247	"cmpwi %%r19, 0\n\t" \
	248	"bne 222b\n\t" \
	249	"333:\n\t" \
	250	RSEQ_INJECT_ASM(4)
	251
	252	#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
	253	, [to_write_memcpy]"r"(_to_write_memcpy), \
	254	[target_memcpy]"r"(_target_memcpy), \
	255	[len_memcpy]"r"(_len_memcpy)
	256
	257	#define RSEQ_FINISH_MEMCPY_CLOBBER() \
	258	, "r18", "r19", "r20", "r21"
	259
	260	#define RSEQ_FINISH_MEMCPY_SCRATCH()
	261
	262	/*
	263	* We use extra registers to hold the input registers, and we don't need to
	264	* save and restore the input registers.
	265	*/
266	#define RSEQ_FINISH_MEMCPY_SETUP() \
267	"mr %%r19, %[len_memcpy]\n\t" \
268	"mr %%r20, %[to_write_memcpy]\n\t" \
269	"mr %%r21, %[target_memcpy]\n\t" \
270
271	#define RSEQ_FINISH_MEMCPY_TEARDOWN()
272
273	#endif /* #else #ifdef __PPC64__ */