[lttng-ust.git] / libringbuffer / rseq-arm.h

/*
 * rseq-arm.h
 *
 * (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#define has_fast_acquire_release()	0
#define has_single_copy_load_64()	1

/*
 * The __rseq_table section can be used by debuggers to better handle
 * single-stepping through the restartable critical sections.
 *
 * Load the immediate value 0 into register r1 right after the ldr
 * instruction to improve instruction-level parallelism: load the
 * constant while the processor is stalled waiting for the load to
 * complete, which is required by the following comparison and branch.
 */

#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
		_failure, _spec_store, _spec_input, \
		_final_store, _final_input, _extra_clobber, \
		_setup, _teardown, _scratch) \
do { \
	_scratch \
	__asm__ __volatile__ goto ( \
		".pushsection __rseq_table, \"aw\"\n\t" \
		".balign 32\n\t" \
		".word 1f, 0x0, 2f, 0x0, 5f, 0x0, 0x0, 0x0\n\t" \
		".popsection\n\t" \
		"1:\n\t" \
		_setup \
		RSEQ_INJECT_ASM(1) \
		"adr r0, 3f\n\t" \
		"str r0, [%[rseq_cs]]\n\t" \
		RSEQ_INJECT_ASM(2) \
		"ldr r0, %[current_event_counter]\n\t" \
		"mov r1, #0\n\t" \
		"cmp %[start_event_counter], r0\n\t" \
		"bne 5f\n\t" \
		RSEQ_INJECT_ASM(3) \
		_spec_store \
		_final_store \
		"2:\n\t" \
		RSEQ_INJECT_ASM(5) \
		"str r1, [%[rseq_cs]]\n\t" \
		_teardown \
		"b 4f\n\t" \
		".balign 32\n\t" \
		"3:\n\t" \
		".word 1b, 0x0, 2b, 0x0, 5f, 0x0, 0x0, 0x0\n\t" \
		"5:\n\t" \
		"mov r1, #0\n\t" \
		"str r1, [%[rseq_cs]]\n\t" \
		_teardown \
		"b %l[failure]\n\t" \
		"4:\n\t" \
		: /* gcc asm goto does not allow outputs */ \
		: [start_event_counter]"r"((_start_value).event_counter), \
		  [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
		  [rseq_cs]"r"(&(_start_value).rseqp->rseq_cs) \
		  _spec_input \
		  _final_input \
		  RSEQ_INJECT_INPUT \
		: "r0", "r1", "memory", "cc" \
		  _extra_clobber \
		  RSEQ_INJECT_CLOBBER \
		: _failure \
	); \
} while (0)

#define RSEQ_FINISH_FINAL_STORE_ASM() \
		"str %[to_write_final], [%[target_final]]\n\t"

#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
		"dmb\n\t" \
		RSEQ_FINISH_FINAL_STORE_ASM()

#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
		, [to_write_final]"r"(_to_write_final), \
		[target_final]"r"(_target_final)

#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
		"str %[to_write_spec], [%[target_spec]]\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
		, [to_write_spec]"r"(_to_write_spec), \
		[target_spec]"r"(_target_spec)

/* TODO: implement a faster memcpy. */
#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
		"cmp %[len_memcpy], #0\n\t" \
		"beq 333f\n\t" \
		"222:\n\t" \
		"ldrb %%r0, [%[to_write_memcpy]]\n\t" \
		"strb %%r0, [%[target_memcpy]]\n\t" \
		"adds %[to_write_memcpy], #1\n\t" \
		"adds %[target_memcpy], #1\n\t" \
		"subs %[len_memcpy], #1\n\t" \
		"bne 222b\n\t" \
		"333:\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
		, [to_write_memcpy]"r"(_to_write_memcpy), \
		[target_memcpy]"r"(_target_memcpy), \
		[len_memcpy]"r"(_len_memcpy), \
		[rseq_scratch0]"m"(rseq_scratch[0]), \
		[rseq_scratch1]"m"(rseq_scratch[1]), \
		[rseq_scratch2]"m"(rseq_scratch[2])

/* We can use r0. */
#define RSEQ_FINISH_MEMCPY_CLOBBER()

#define RSEQ_FINISH_MEMCPY_SCRATCH() \
		uint32_t rseq_scratch[3];

/*
 * We need to save and restore those input registers so they can be
 * modified within the assembly.
 */
#define RSEQ_FINISH_MEMCPY_SETUP() \
		"str %[to_write_memcpy], %[rseq_scratch0]\n\t" \
		"str %[target_memcpy], %[rseq_scratch1]\n\t" \
		"str %[len_memcpy], %[rseq_scratch2]\n\t"

#define RSEQ_FINISH_MEMCPY_TEARDOWN() \
		"ldr %[len_memcpy], %[rseq_scratch2]\n\t" \
		"ldr %[target_memcpy], %[rseq_scratch1]\n\t" \
		"ldr %[to_write_memcpy], %[rseq_scratch0]\n\t"
Commit	Line	Data
b76e5200 MD	1	/*
	2	* rseq-arm.h
	3	*
	4	* (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	5	*
	6	* Permission is hereby granted, free of charge, to any person obtaining a copy
	7	* of this software and associated documentation files (the "Software"), to deal
	8	* in the Software without restriction, including without limitation the rights
	9	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	10	* copies of the Software, and to permit persons to whom the Software is
	11	* furnished to do so, subject to the following conditions:
	12	*
	13	* The above copyright notice and this permission notice shall be included in
	14	* all copies or substantial portions of the Software.
	15	*
	16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	19	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	21	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	22	* SOFTWARE.
	23	*/
	24
b76e5200 MD	25	#define has_fast_acquire_release() 0
	26	#define has_single_copy_load_64() 1
	27
	28	/*
	29	* The __rseq_table section can be used by debuggers to better handle
	30	* single-stepping through the restartable critical sections.
	31	*
	32	* Load the immediate value 0 into register r1 right after the ldr
	33	* instruction to improve instruction-level parallelism: load the
	34	* constant while the processor is stalled waiting for the load to
	35	* complete, which is required by the following comparison and branch.
	36	*/
	37
	38	#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
	39	_failure, _spec_store, _spec_input, \
	40	_final_store, _final_input, _extra_clobber, \
	41	_setup, _teardown, _scratch) \
	42	do { \
	43	_scratch \
	44	__asm__ __volatile__ goto ( \
	45	".pushsection __rseq_table, \"aw\"\n\t" \
	46	".balign 32\n\t" \
	47	".word 1f, 0x0, 2f, 0x0, 5f, 0x0, 0x0, 0x0\n\t" \
	48	".popsection\n\t" \
	49	"1:\n\t" \
	50	_setup \
	51	RSEQ_INJECT_ASM(1) \
	52	"adr r0, 3f\n\t" \
	53	"str r0, [%[rseq_cs]]\n\t" \
	54	RSEQ_INJECT_ASM(2) \
	55	"ldr r0, %[current_event_counter]\n\t" \
	56	"mov r1, #0\n\t" \
	57	"cmp %[start_event_counter], r0\n\t" \
	58	"bne 5f\n\t" \
	59	RSEQ_INJECT_ASM(3) \
	60	_spec_store \
	61	_final_store \
	62	"2:\n\t" \
	63	RSEQ_INJECT_ASM(5) \
	64	"str r1, [%[rseq_cs]]\n\t" \
	65	_teardown \
	66	"b 4f\n\t" \
	67	".balign 32\n\t" \
	68	"3:\n\t" \
	69	".word 1b, 0x0, 2b, 0x0, 5f, 0x0, 0x0, 0x0\n\t" \
	70	"5:\n\t" \
	71	"mov r1, #0\n\t" \
	72	"str r1, [%[rseq_cs]]\n\t" \
	73	_teardown \
	74	"b %l[failure]\n\t" \
	75	"4:\n\t" \
26cc635c	76	: /* gcc asm goto does not allow outputs */ \
b76e5200 MD	77	: [start_event_counter]"r"((_start_value).event_counter), \
	78	[current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
	79	[rseq_cs]"r"(&(_start_value).rseqp->rseq_cs) \
	80	_spec_input \
	81	_final_input \
	82	RSEQ_INJECT_INPUT \
	83	: "r0", "r1", "memory", "cc" \
	84	_extra_clobber \
	85	RSEQ_INJECT_CLOBBER \
	86	: _failure \
	87	); \
	88	} while (0)
	89
	90	#define RSEQ_FINISH_FINAL_STORE_ASM() \
	91	"str %[to_write_final], [%[target_final]]\n\t"
	92
	93	#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
	94	"dmb\n\t" \
	95	RSEQ_FINISH_FINAL_STORE_ASM()
	96
	97	#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
	98	, [to_write_final]"r"(_to_write_final), \
	99	[target_final]"r"(_target_final)
	100
	101	#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
	102	"str %[to_write_spec], [%[target_spec]]\n\t" \
	103	RSEQ_INJECT_ASM(4)
	104
	105	#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
	106	, [to_write_spec]"r"(_to_write_spec), \
	107	[target_spec]"r"(_target_spec)
	108
	109	/* TODO: implement a faster memcpy. */
	110	#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
	111	"cmp %[len_memcpy], #0\n\t" \
	112	"beq 333f\n\t" \
	113	"222:\n\t" \
	114	"ldrb %%r0, [%[to_write_memcpy]]\n\t" \
	115	"strb %%r0, [%[target_memcpy]]\n\t" \
	116	"adds %[to_write_memcpy], #1\n\t" \
	117	"adds %[target_memcpy], #1\n\t" \
	118	"subs %[len_memcpy], #1\n\t" \
	119	"bne 222b\n\t" \
	120	"333:\n\t" \
	121	RSEQ_INJECT_ASM(4)
	122
	123	#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
	124	, [to_write_memcpy]"r"(_to_write_memcpy), \
	125	[target_memcpy]"r"(_target_memcpy), \
	126	[len_memcpy]"r"(_len_memcpy), \
	127	[rseq_scratch0]"m"(rseq_scratch[0]), \
	128	[rseq_scratch1]"m"(rseq_scratch[1]), \
	129	[rseq_scratch2]"m"(rseq_scratch[2])
	130
	131	/* We can use r0. */
	132	#define RSEQ_FINISH_MEMCPY_CLOBBER()
	133
	134	#define RSEQ_FINISH_MEMCPY_SCRATCH() \
	135	uint32_t rseq_scratch[3];
	136
	137	/*
	138	* We need to save and restore those input registers so they can be
	139	* modified within the assembly.
	140	*/
141	#define RSEQ_FINISH_MEMCPY_SETUP() \
142	"str %[to_write_memcpy], %[rseq_scratch0]\n\t" \
143	"str %[target_memcpy], %[rseq_scratch1]\n\t" \
144	"str %[len_memcpy], %[rseq_scratch2]\n\t"
145
146	#define RSEQ_FINISH_MEMCPY_TEARDOWN() \
147	"ldr %[len_memcpy], %[rseq_scratch2]\n\t" \
148	"ldr %[target_memcpy], %[rseq_scratch1]\n\t" \
149	"ldr %[to_write_memcpy], %[rseq_scratch0]\n\t"