1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
3 * counter/counter-api.h
5 * LTTng Counters API, requiring counter/config.h
7 * Copyright (C) 2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
10 #ifndef _LTTNG_COUNTER_API_H
11 #define _LTTNG_COUNTER_API_H
13 #include <linux/types.h>
14 #include <linux/percpu.h>
15 #include <linux/bitops.h>
16 #include <counter/counter.h>
17 #include <counter/counter-internal.h>
20 * Using unsigned arithmetic because overflow is defined.
22 static inline int __lttng_counter_add(const struct lib_counter_config
*config
,
23 enum lib_counter_config_alloc alloc
,
24 enum lib_counter_config_sync sync
,
25 struct lib_counter
*counter
,
26 const size_t *dimension_indexes
, int64_t v
,
30 bool overflow
= false, underflow
= false;
31 struct lib_counter_layout
*layout
;
34 if (unlikely(lttng_counter_validate_indexes(config
, counter
, dimension_indexes
)))
36 index
= lttng_counter_get_index(config
, counter
, dimension_indexes
);
39 case COUNTER_ALLOC_PER_CPU
:
40 layout
= per_cpu_ptr(counter
->percpu_counters
, smp_processor_id());
42 case COUNTER_ALLOC_GLOBAL
:
43 layout
= &counter
->global_counters
;
49 switch (config
->counter_size
) {
50 case COUNTER_SIZE_8_BIT
:
52 int8_t *int_p
= (int8_t *) layout
->counters
+ index
;
54 int8_t global_sum_step
= counter
->global_sum_step
.s8
;
58 case COUNTER_SYNC_PER_CPU
:
63 n
= (int8_t) ((uint8_t) old
+ (uint8_t) v
);
64 if (unlikely(n
> (int8_t) global_sum_step
))
65 move_sum
= (int8_t) global_sum_step
/ 2;
66 else if (unlikely(n
< -(int8_t) global_sum_step
))
67 move_sum
= -((int8_t) global_sum_step
/ 2);
69 res
= cmpxchg_local(int_p
, old
, n
);
73 case COUNTER_SYNC_GLOBAL
:
77 n
= (int8_t) ((uint8_t) old
+ (uint8_t) v
);
78 res
= cmpxchg(int_p
, old
, n
);
83 if (v
> 0 && (v
>= U8_MAX
|| n
< old
))
85 else if (v
< 0 && (v
<= -U8_MAX
|| n
> old
))
89 case COUNTER_SIZE_16_BIT
:
91 int16_t *int_p
= (int16_t *) layout
->counters
+ index
;
93 int16_t global_sum_step
= counter
->global_sum_step
.s16
;
97 case COUNTER_SYNC_PER_CPU
:
102 n
= (int16_t) ((uint16_t) old
+ (uint16_t) v
);
103 if (unlikely(n
> (int16_t) global_sum_step
))
104 move_sum
= (int16_t) global_sum_step
/ 2;
105 else if (unlikely(n
< -(int16_t) global_sum_step
))
106 move_sum
= -((int16_t) global_sum_step
/ 2);
108 res
= cmpxchg_local(int_p
, old
, n
);
109 } while (old
!= res
);
112 case COUNTER_SYNC_GLOBAL
:
116 n
= (int16_t) ((uint16_t) old
+ (uint16_t) v
);
117 res
= cmpxchg(int_p
, old
, n
);
118 } while (old
!= res
);
122 if (v
> 0 && (v
>= U16_MAX
|| n
< old
))
124 else if (v
< 0 && (v
<= -U16_MAX
|| n
> old
))
128 case COUNTER_SIZE_32_BIT
:
130 int32_t *int_p
= (int32_t *) layout
->counters
+ index
;
132 int32_t global_sum_step
= counter
->global_sum_step
.s32
;
136 case COUNTER_SYNC_PER_CPU
:
141 n
= (int32_t) ((uint32_t) old
+ (uint32_t) v
);
142 if (unlikely(n
> (int32_t) global_sum_step
))
143 move_sum
= (int32_t) global_sum_step
/ 2;
144 else if (unlikely(n
< -(int32_t) global_sum_step
))
145 move_sum
= -((int32_t) global_sum_step
/ 2);
147 res
= cmpxchg_local(int_p
, old
, n
);
148 } while (old
!= res
);
151 case COUNTER_SYNC_GLOBAL
:
155 n
= (int32_t) ((uint32_t) old
+ (uint32_t) v
);
156 res
= cmpxchg(int_p
, old
, n
);
157 } while (old
!= res
);
161 if (v
> 0 && (v
>= U32_MAX
|| n
< old
))
163 else if (v
< 0 && (v
<= -U32_MAX
|| n
> old
))
167 #if BITS_PER_LONG == 64
168 case COUNTER_SIZE_64_BIT
:
170 int64_t *int_p
= (int64_t *) layout
->counters
+ index
;
172 int64_t global_sum_step
= counter
->global_sum_step
.s64
;
176 case COUNTER_SYNC_PER_CPU
:
181 n
= (int64_t) ((uint64_t) old
+ (uint64_t) v
);
182 if (unlikely(n
> (int64_t) global_sum_step
))
183 move_sum
= (int64_t) global_sum_step
/ 2;
184 else if (unlikely(n
< -(int64_t) global_sum_step
))
185 move_sum
= -((int64_t) global_sum_step
/ 2);
187 res
= cmpxchg_local(int_p
, old
, n
);
188 } while (old
!= res
);
191 case COUNTER_SYNC_GLOBAL
:
195 n
= (int64_t) ((uint64_t) old
+ (uint64_t) v
);
196 res
= cmpxchg(int_p
, old
, n
);
197 } while (old
!= res
);
201 if (v
> 0 && n
< old
)
203 else if (v
< 0 && n
> old
)
211 if (unlikely(overflow
&& !test_bit(index
, layout
->overflow_bitmap
)))
212 set_bit(index
, layout
->overflow_bitmap
);
213 else if (unlikely(underflow
&& !test_bit(index
, layout
->underflow_bitmap
)))
214 set_bit(index
, layout
->underflow_bitmap
);
216 *remainder
= move_sum
;
220 static inline int __lttng_counter_add_percpu(const struct lib_counter_config
*config
,
221 struct lib_counter
*counter
,
222 const size_t *dimension_indexes
, int64_t v
)
227 ret
= __lttng_counter_add(config
, COUNTER_ALLOC_PER_CPU
, config
->sync
,
228 counter
, dimension_indexes
, v
, &move_sum
);
231 if (unlikely(move_sum
))
232 return __lttng_counter_add(config
, COUNTER_ALLOC_GLOBAL
, COUNTER_SYNC_GLOBAL
,
233 counter
, dimension_indexes
, move_sum
, NULL
);
237 static inline int __lttng_counter_add_global(const struct lib_counter_config
*config
,
238 struct lib_counter
*counter
,
239 const size_t *dimension_indexes
, int64_t v
)
241 return __lttng_counter_add(config
, COUNTER_ALLOC_GLOBAL
, config
->sync
, counter
,
242 dimension_indexes
, v
, NULL
);
245 static inline int lttng_counter_add(const struct lib_counter_config
*config
,
246 struct lib_counter
*counter
,
247 const size_t *dimension_indexes
, int64_t v
)
249 switch (config
->alloc
) {
250 case COUNTER_ALLOC_PER_CPU
: /* Fallthrough */
251 case COUNTER_ALLOC_PER_CPU
| COUNTER_ALLOC_GLOBAL
:
252 return __lttng_counter_add_percpu(config
, counter
, dimension_indexes
, v
);
253 case COUNTER_ALLOC_GLOBAL
:
254 return __lttng_counter_add_global(config
, counter
, dimension_indexes
, v
);
260 static inline int lttng_counter_inc(const struct lib_counter_config
*config
,
261 struct lib_counter
*counter
,
262 const size_t *dimension_indexes
)
264 return lttng_counter_add(config
, counter
, dimension_indexes
, 1);
267 static inline int lttng_counter_dec(const struct lib_counter_config
*config
,
268 struct lib_counter
*counter
,
269 const size_t *dimension_indexes
)
271 return lttng_counter_add(config
, counter
, dimension_indexes
, -1);
274 #endif /* _LTTNG_COUNTER_API_H */