[deliverable/linux.git] / tools / testing / selftests / rseq / rseq-x86.h

/*
 * rseq-x86.h
 *
 * (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#ifdef __x86_64__

#define smp_mb()	__asm__ __volatile__ ("mfence" : : : "memory")
#define smp_rmb()	barrier()
#define smp_wmb()	barrier()

#define smp_load_acquire(p)						\
__extension__ ({							\
	__typeof(*p) ____p1 = READ_ONCE(*p);				\
	barrier();							\
	____p1;								\
})

#define smp_acquire__after_ctrl_dep()	smp_rmb()

#define smp_store_release(p, v)						\
do {									\
	barrier();							\
	WRITE_ONCE(*p, v);						\
} while (0)

#define has_fast_acquire_release()	1
#define has_single_copy_load_64()	1

/*
 * The __rseq_table section can be used by debuggers to better handle
 * single-stepping through the restartable critical sections.
 */
#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
		_failure, _spec_store, _spec_input, \
		_final_store, _final_input, _extra_clobber, \
		_setup, _teardown, _scratch) \
do { \
	_scratch \
	__asm__ __volatile__ goto ( \
		".pushsection __rseq_table, \"aw\"\n\t" \
		".balign 32\n\t" \
		"3:\n\t" \
		".quad 1f, 2f, 4f, 0x0\n\t" \
		".popsection\n\t" \
		"1:\n\t" \
		_setup \
		RSEQ_INJECT_ASM(1) \
		"leaq 3b(%%rip), %%rax\n\t" \
		"movq %%rax, %[rseq_cs]\n\t" \
		RSEQ_INJECT_ASM(2) \
		"cmpl %[start_event_counter], %[current_event_counter]\n\t" \
		"jnz 4f\n\t" \
		RSEQ_INJECT_ASM(3) \
		_spec_store \
		_final_store \
		"2:\n\t" \
		RSEQ_INJECT_ASM(5) \
		_teardown \
		".pushsection __rseq_failure, \"a\"\n\t" \
		"4:\n\t" \
		_teardown \
		"jmp %l[failure]\n\t" \
		".popsection\n\t" \
		: /* gcc asm goto does not allow outputs */ \
		: [start_event_counter]"r"((_start_value).event_counter), \
		  [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
		  [rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
		  _spec_input \
		  _final_input \
		  RSEQ_INJECT_INPUT \
		: "memory", "cc", "rax" \
		  _extra_clobber \
		  RSEQ_INJECT_CLOBBER \
		: _failure \
	); \
} while (0)

#define RSEQ_FINISH_FINAL_STORE_ASM() \
		"movq %[to_write_final], %[target_final]\n\t"

/* x86-64 is TSO */
#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
		RSEQ_FINISH_FINAL_STORE_ASM()

#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
		, [to_write_final]"r"(_to_write_final), \
		[target_final]"m"(*(_target_final))

#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
		"movq %[to_write_spec], %[target_spec]\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
		, [to_write_spec]"r"(_to_write_spec), \
		[target_spec]"m"(*(_target_spec))

/* TODO: implement a faster memcpy. */
#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
		"test %[len_memcpy], %[len_memcpy]\n\t" \
		"jz 333f\n\t" \
		"222:\n\t" \
		"movb (%[to_write_memcpy]), %%al\n\t" \
		"movb %%al, (%[target_memcpy])\n\t" \
		"inc %[to_write_memcpy]\n\t" \
		"inc %[target_memcpy]\n\t" \
		"dec %[len_memcpy]\n\t" \
		"jnz 222b\n\t" \
		"333:\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
		, [to_write_memcpy]"r"(_to_write_memcpy), \
		[target_memcpy]"r"(_target_memcpy), \
		[len_memcpy]"r"(_len_memcpy), \
		[rseq_scratch0]"m"(rseq_scratch[0]), \
		[rseq_scratch1]"m"(rseq_scratch[1]), \
		[rseq_scratch2]"m"(rseq_scratch[2])

#define RSEQ_FINISH_MEMCPY_CLOBBER()	\
		, "rax"

#define RSEQ_FINISH_MEMCPY_SCRATCH() \
		uint64_t rseq_scratch[3];

/*
 * We need to save and restore those input registers so they can be
 * modified within the assembly.
 */
#define RSEQ_FINISH_MEMCPY_SETUP() \
		"movq %[to_write_memcpy], %[rseq_scratch0]\n\t" \
		"movq %[target_memcpy], %[rseq_scratch1]\n\t" \
		"movq %[len_memcpy], %[rseq_scratch2]\n\t"

#define RSEQ_FINISH_MEMCPY_TEARDOWN() \
		"movq %[rseq_scratch2], %[len_memcpy]\n\t" \
		"movq %[rseq_scratch1], %[target_memcpy]\n\t" \
		"movq %[rseq_scratch0], %[to_write_memcpy]\n\t"

#elif __i386__

/*
 * Support older 32-bit architectures that do not implement fence
 * instructions.
 */
#define smp_mb()	\
	__asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
#define smp_rmb()	\
	__asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
#define smp_wmb()	\
	__asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")

#define smp_load_acquire(p)						\
__extension__ ({							\
	__typeof(*p) ____p1 = READ_ONCE(*p);				\
	smp_mb();							\
	____p1;								\
})

#define smp_acquire__after_ctrl_dep()	smp_rmb()

#define smp_store_release(p, v)						\
do {									\
	smp_mb();							\
	WRITE_ONCE(*p, v);						\
} while (0)

#define has_fast_acquire_release()	0
#define has_single_copy_load_64()	0

/*
 * Use eax as scratch register and take memory operands as input to
 * lessen register pressure. Especially needed when compiling
 * do_rseq_memcpy() in O0.
 */
#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
		_failure, _spec_store, _spec_input, \
		_final_store, _final_input, _extra_clobber, \
		_setup, _teardown, _scratch) \
do { \
	_scratch \
	__asm__ __volatile__ goto ( \
		".pushsection __rseq_table, \"aw\"\n\t" \
		".balign 32\n\t" \
		"3:\n\t" \
		".long 1f, 0x0, 2f, 0x0, 4f, 0x0, 0x0, 0x0\n\t" \
		".popsection\n\t" \
		"1:\n\t" \
		_setup \
		RSEQ_INJECT_ASM(1) \
		"movl $3b, %[rseq_cs]\n\t" \
		RSEQ_INJECT_ASM(2) \
		"movl %[start_event_counter], %%eax\n\t" \
		"cmpl %%eax, %[current_event_counter]\n\t" \
		"jnz 4f\n\t" \
		RSEQ_INJECT_ASM(3) \
		_spec_store \
		_final_store \
		"2:\n\t" \
		RSEQ_INJECT_ASM(5) \
		_teardown \
		".pushsection __rseq_failure, \"a\"\n\t" \
		"4:\n\t" \
		_teardown \
		"jmp %l[failure]\n\t" \
		".popsection\n\t" \
		: /* gcc asm goto does not allow outputs */ \
		: [start_event_counter]"m"((_start_value).event_counter), \
		  [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
		  [rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
		  _spec_input \
		  _final_input \
		  RSEQ_INJECT_INPUT \
		: "memory", "cc", "eax" \
		  _extra_clobber \
		  RSEQ_INJECT_CLOBBER \
		: _failure \
	); \
} while (0)

#define RSEQ_FINISH_FINAL_STORE_ASM() \
		"movl %[to_write_final], %%eax\n\t" \
		"movl %%eax, %[target_final]\n\t"

#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
		"lock; addl $0,0(%%esp)\n\t" \
		RSEQ_FINISH_FINAL_STORE_ASM()

#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
		, [to_write_final]"m"(_to_write_final), \
		[target_final]"m"(*(_target_final))

#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
		"movl %[to_write_spec], %%eax\n\t" \
		"movl %%eax, %[target_spec]\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
		, [to_write_spec]"m"(_to_write_spec), \
		[target_spec]"m"(*(_target_spec))

/* TODO: implement a faster memcpy. */
#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
		"movl %[len_memcpy], %%eax\n\t" \
		"test %%eax, %%eax\n\t" \
		"jz 333f\n\t" \
		"222:\n\t" \
		"movb (%[to_write_memcpy]), %%al\n\t" \
		"movb %%al, (%[target_memcpy])\n\t" \
		"inc %[to_write_memcpy]\n\t" \
		"inc %[target_memcpy]\n\t" \
		"decl %[rseq_scratch2]\n\t" \
		"jnz 222b\n\t" \
		"333:\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
		, [to_write_memcpy]"r"(_to_write_memcpy), \
		[target_memcpy]"r"(_target_memcpy), \
		[len_memcpy]"m"(_len_memcpy), \
		[rseq_scratch0]"m"(rseq_scratch[0]), \
		[rseq_scratch1]"m"(rseq_scratch[1]), \
		[rseq_scratch2]"m"(rseq_scratch[2])

#define RSEQ_FINISH_MEMCPY_CLOBBER()

#define RSEQ_FINISH_MEMCPY_SCRATCH() \
		uint32_t rseq_scratch[3];

/*
 * We need to save and restore those input registers so they can be
 * modified within the assembly.
 */
#define RSEQ_FINISH_MEMCPY_SETUP() \
		"movl %[to_write_memcpy], %[rseq_scratch0]\n\t" \
		"movl %[target_memcpy], %[rseq_scratch1]\n\t" \
		"movl %[len_memcpy], %%eax\n\t" \
		"movl %%eax, %[rseq_scratch2]\n\t"

#define RSEQ_FINISH_MEMCPY_TEARDOWN() \
		"movl %[rseq_scratch1], %[target_memcpy]\n\t" \
		"movl %[rseq_scratch0], %[to_write_memcpy]\n\t"

#endif
Commit	Line	Data
b54c5158 MD	1	/*
	2	* rseq-x86.h
	3	*
	4	* (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	5	*
	6	* Permission is hereby granted, free of charge, to any person obtaining a copy
	7	* of this software and associated documentation files (the "Software"), to deal
	8	* in the Software without restriction, including without limitation the rights
	9	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	10	* copies of the Software, and to permit persons to whom the Software is
	11	* furnished to do so, subject to the following conditions:
	12	*
	13	* The above copyright notice and this permission notice shall be included in
	14	* all copies or substantial portions of the Software.
	15	*
	16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	19	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	21	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	22	* SOFTWARE.
	23	*/
	24
	25	#ifdef __x86_64__
	26
	27	#define smp_mb() __asm__ __volatile__ ("mfence" : : : "memory")
	28	#define smp_rmb() barrier()
	29	#define smp_wmb() barrier()
	30
	31	#define smp_load_acquire(p) \
	32	__extension__ ({ \
	33	__typeof(p) ____p1 = READ_ONCE(p); \
	34	barrier(); \
	35	____p1; \
	36	})
	37
	38	#define smp_acquire__after_ctrl_dep() smp_rmb()
	39
	40	#define smp_store_release(p, v) \
	41	do { \
	42	barrier(); \
	43	WRITE_ONCE(*p, v); \
	44	} while (0)
	45
	46	#define has_fast_acquire_release() 1
	47	#define has_single_copy_load_64() 1
	48
	49	/*
	50	* The __rseq_table section can be used by debuggers to better handle
	51	* single-stepping through the restartable critical sections.
	52	*/
	53	#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
	54	_failure, _spec_store, _spec_input, \
	55	_final_store, _final_input, _extra_clobber, \
	56	_setup, _teardown, _scratch) \
	57	do { \
	58	_scratch \
	59	__asm__ __volatile__ goto ( \
	60	".pushsection __rseq_table, \"aw\"\n\t" \
	61	".balign 32\n\t" \
	62	"3:\n\t" \
	63	".quad 1f, 2f, 4f, 0x0\n\t" \
	64	".popsection\n\t" \
65	"1:\n\t" \
66	_setup \
67	RSEQ_INJECT_ASM(1) \
f88ff702 MD	68	"leaq 3b(%%rip), %%rax\n\t" \
f88ff702 MD	69	"movq %%rax, %[rseq_cs]\n\t" \
b54c5158 MD	70	RSEQ_INJECT_ASM(2) \
	71	"cmpl %[start_event_counter], %[current_event_counter]\n\t" \
	72	"jnz 4f\n\t" \
	73	RSEQ_INJECT_ASM(3) \
	74	_spec_store \
	75	_final_store \
	76	"2:\n\t" \
	77	RSEQ_INJECT_ASM(5) \
b54c5158 MD	78	_teardown \
	79	".pushsection __rseq_failure, \"a\"\n\t" \
	80	"4:\n\t" \
b54c5158 MD	81	_teardown \
	82	"jmp %l[failure]\n\t" \
	83	".popsection\n\t" \
	84	: /* gcc asm goto does not allow outputs */ \
	85	: [start_event_counter]"r"((_start_value).event_counter), \
	86	[current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
	87	[rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
	88	_spec_input \
	89	_final_input \
	90	RSEQ_INJECT_INPUT \
f88ff702	91	: "memory", "cc", "rax" \
b54c5158 MD	92	_extra_clobber \
	93	RSEQ_INJECT_CLOBBER \
	94	: _failure \
	95	); \
	96	} while (0)
	97
	98	#define RSEQ_FINISH_FINAL_STORE_ASM() \
	99	"movq %[to_write_final], %[target_final]\n\t"
	100
	101	/* x86-64 is TSO */
	102	#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
	103	RSEQ_FINISH_FINAL_STORE_ASM()
	104
	105	#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
	106	, [to_write_final]"r"(_to_write_final), \
	107	[target_final]"m"(*(_target_final))
	108
	109	#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
	110	"movq %[to_write_spec], %[target_spec]\n\t" \
	111	RSEQ_INJECT_ASM(4)
	112
	113	#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
	114	, [to_write_spec]"r"(_to_write_spec), \
	115	[target_spec]"m"(*(_target_spec))
	116
	117	/* TODO: implement a faster memcpy. */
	118	#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
	119	"test %[len_memcpy], %[len_memcpy]\n\t" \
	120	"jz 333f\n\t" \
	121	"222:\n\t" \
	122	"movb (%[to_write_memcpy]), %%al\n\t" \
	123	"movb %%al, (%[target_memcpy])\n\t" \
	124	"inc %[to_write_memcpy]\n\t" \
	125	"inc %[target_memcpy]\n\t" \
	126	"dec %[len_memcpy]\n\t" \
	127	"jnz 222b\n\t" \
	128	"333:\n\t" \
	129	RSEQ_INJECT_ASM(4)
	130
	131	#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
	132	, [to_write_memcpy]"r"(_to_write_memcpy), \
	133	[target_memcpy]"r"(_target_memcpy), \
	134	[len_memcpy]"r"(_len_memcpy), \
	135	[rseq_scratch0]"m"(rseq_scratch[0]), \
	136	[rseq_scratch1]"m"(rseq_scratch[1]), \
	137	[rseq_scratch2]"m"(rseq_scratch[2])
	138
	139	#define RSEQ_FINISH_MEMCPY_CLOBBER() \
	140	, "rax"
	141
	142	#define RSEQ_FINISH_MEMCPY_SCRATCH() \
	143	uint64_t rseq_scratch[3];
	144
	145	/*
	146	* We need to save and restore those input registers so they can be
	147	* modified within the assembly.
	148	*/
	149	#define RSEQ_FINISH_MEMCPY_SETUP() \
	150	"movq %[to_write_memcpy], %[rseq_scratch0]\n\t" \
	151	"movq %[target_memcpy], %[rseq_scratch1]\n\t" \
	152	"movq %[len_memcpy], %[rseq_scratch2]\n\t"
	153
	154	#define RSEQ_FINISH_MEMCPY_TEARDOWN() \
	155	"movq %[rseq_scratch2], %[len_memcpy]\n\t" \
156	"movq %[rseq_scratch1], %[target_memcpy]\n\t" \
157	"movq %[rseq_scratch0], %[to_write_memcpy]\n\t"
158
159	#elif __i386__
160
161	/*
162	* Support older 32-bit architectures that do not implement fence
163	* instructions.
164	*/
165	#define smp_mb() \
166	__asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
167	#define smp_rmb() \
168	__asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
169	#define smp_wmb() \
170	__asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
171
172	#define smp_load_acquire(p) \
173	__extension__ ({ \
174	__typeof(p) ____p1 = READ_ONCE(p); \
175	smp_mb(); \
176	____p1; \
177	})
178
179	#define smp_acquire__after_ctrl_dep() smp_rmb()
180
181	#define smp_store_release(p, v) \
182	do { \
183	smp_mb(); \
184	WRITE_ONCE(*p, v); \
185	} while (0)
186
187	#define has_fast_acquire_release() 0
188	#define has_single_copy_load_64() 0
189
190	/*
191	* Use eax as scratch register and take memory operands as input to
192	* lessen register pressure. Especially needed when compiling
193	* do_rseq_memcpy() in O0.
194	*/
195	#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
196	_failure, _spec_store, _spec_input, \
197	_final_store, _final_input, _extra_clobber, \
198	_setup, _teardown, _scratch) \
199	do { \
200	_scratch \
201	__asm__ __volatile__ goto ( \
202	".pushsection __rseq_table, \"aw\"\n\t" \
203	".balign 32\n\t" \
204	"3:\n\t" \
205	".long 1f, 0x0, 2f, 0x0, 4f, 0x0, 0x0, 0x0\n\t" \
206	".popsection\n\t" \
207	"1:\n\t" \
208	_setup \
209	RSEQ_INJECT_ASM(1) \
210	"movl $3b, %[rseq_cs]\n\t" \
211	RSEQ_INJECT_ASM(2) \
212	"movl %[start_event_counter], %%eax\n\t" \
213	"cmpl %%eax, %[current_event_counter]\n\t" \
214	"jnz 4f\n\t" \
215	RSEQ_INJECT_ASM(3) \
216	_spec_store \
217	_final_store \
218	"2:\n\t" \
219	RSEQ_INJECT_ASM(5) \
b54c5158 MD	220	_teardown \
	221	".pushsection __rseq_failure, \"a\"\n\t" \
	222	"4:\n\t" \
b54c5158 MD	223	_teardown \
	224	"jmp %l[failure]\n\t" \
	225	".popsection\n\t" \
	226	: /* gcc asm goto does not allow outputs */ \
	227	: [start_event_counter]"m"((_start_value).event_counter), \
	228	[current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
	229	[rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
	230	_spec_input \
	231	_final_input \
	232	RSEQ_INJECT_INPUT \
	233	: "memory", "cc", "eax" \
	234	_extra_clobber \
	235	RSEQ_INJECT_CLOBBER \
	236	: _failure \
	237	); \
	238	} while (0)
	239
	240	#define RSEQ_FINISH_FINAL_STORE_ASM() \
	241	"movl %[to_write_final], %%eax\n\t" \
	242	"movl %%eax, %[target_final]\n\t"
	243
	244	#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
	245	"lock; addl $0,0(%%esp)\n\t" \
	246	RSEQ_FINISH_FINAL_STORE_ASM()
	247
	248	#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
	249	, [to_write_final]"m"(_to_write_final), \
	250	[target_final]"m"(*(_target_final))
	251
	252	#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
	253	"movl %[to_write_spec], %%eax\n\t" \
	254	"movl %%eax, %[target_spec]\n\t" \
	255	RSEQ_INJECT_ASM(4)
	256
	257	#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
	258	, [to_write_spec]"m"(_to_write_spec), \
	259	[target_spec]"m"(*(_target_spec))
	260
	261	/* TODO: implement a faster memcpy. */
	262	#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
	263	"movl %[len_memcpy], %%eax\n\t" \
	264	"test %%eax, %%eax\n\t" \
	265	"jz 333f\n\t" \
	266	"222:\n\t" \
	267	"movb (%[to_write_memcpy]), %%al\n\t" \
	268	"movb %%al, (%[target_memcpy])\n\t" \
	269	"inc %[to_write_memcpy]\n\t" \
	270	"inc %[target_memcpy]\n\t" \
	271	"decl %[rseq_scratch2]\n\t" \
	272	"jnz 222b\n\t" \
	273	"333:\n\t" \
	274	RSEQ_INJECT_ASM(4)
	275
	276	#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
	277	, [to_write_memcpy]"r"(_to_write_memcpy), \
	278	[target_memcpy]"r"(_target_memcpy), \
	279	[len_memcpy]"m"(_len_memcpy), \
	280	[rseq_scratch0]"m"(rseq_scratch[0]), \
	281	[rseq_scratch1]"m"(rseq_scratch[1]), \
	282	[rseq_scratch2]"m"(rseq_scratch[2])
	283
	284	#define RSEQ_FINISH_MEMCPY_CLOBBER()
	285
	286	#define RSEQ_FINISH_MEMCPY_SCRATCH() \
287	uint32_t rseq_scratch[3];
288
289	/*
290	* We need to save and restore those input registers so they can be
291	* modified within the assembly.
292	*/
293	#define RSEQ_FINISH_MEMCPY_SETUP() \
294	"movl %[to_write_memcpy], %[rseq_scratch0]\n\t" \
295	"movl %[target_memcpy], %[rseq_scratch1]\n\t" \
296	"movl %[len_memcpy], %%eax\n\t" \
297	"movl %%eax, %[rseq_scratch2]\n\t"
298
299	#define RSEQ_FINISH_MEMCPY_TEARDOWN() \
300	"movl %[rseq_scratch1], %[target_memcpy]\n\t" \
301	"movl %[rseq_scratch0], %[to_write_memcpy]\n\t"
302
303	#endif