Commit | Line | Data |
---|---|---|
8256e47c MD |
1 | /* |
2 | * Copyright (C) 2007 Mathieu Desnoyers | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License as published by | |
6 | * the Free Software Foundation; either version 2 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | */ | |
18 | #include <linux/module.h> | |
19 | #include <linux/mutex.h> | |
20 | #include <linux/types.h> | |
21 | #include <linux/jhash.h> | |
22 | #include <linux/list.h> | |
23 | #include <linux/rcupdate.h> | |
24 | #include <linux/marker.h> | |
25 | #include <linux/err.h> | |
26 | ||
27 | extern struct marker __start___markers[]; | |
28 | extern struct marker __stop___markers[]; | |
29 | ||
fb40bd78 MD |
30 | /* Set to 1 to enable marker debug output */ |
31 | const int marker_debug; | |
32 | ||
8256e47c | 33 | /* |
314de8a9 | 34 | * markers_mutex nests inside module_mutex. Markers mutex protects the builtin |
fb40bd78 | 35 | * and module markers and the hash table. |
8256e47c MD |
36 | */ |
37 | static DEFINE_MUTEX(markers_mutex); | |
38 | ||
8256e47c MD |
39 | /* |
40 | * Marker hash table, containing the active markers. | |
41 | * Protected by module_mutex. | |
42 | */ | |
43 | #define MARKER_HASH_BITS 6 | |
44 | #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) | |
45 | ||
fb40bd78 MD |
46 | /* |
47 | * Note about RCU : | |
48 | * It is used to make sure every handler has finished using its private data | |
49 | * between two consecutive operation (add or remove) on a given marker. It is | |
50 | * also used to delay the free of multiple probes array until a quiescent state | |
51 | * is reached. | |
52 | * marker entries modifications are protected by the markers_mutex. | |
53 | */ | |
8256e47c MD |
54 | struct marker_entry { |
55 | struct hlist_node hlist; | |
56 | char *format; | |
fb40bd78 MD |
57 | void (*call)(const struct marker *mdata, /* Probe wrapper */ |
58 | void *call_private, const char *fmt, ...); | |
59 | struct marker_probe_closure single; | |
60 | struct marker_probe_closure *multi; | |
8256e47c | 61 | int refcount; /* Number of times armed. 0 if disarmed. */ |
fb40bd78 MD |
62 | struct rcu_head rcu; |
63 | void *oldptr; | |
de4fc64f HH |
64 | unsigned char rcu_pending:1; |
65 | unsigned char ptype:1; | |
8256e47c MD |
66 | char name[0]; /* Contains name'\0'format'\0' */ |
67 | }; | |
68 | ||
69 | static struct hlist_head marker_table[MARKER_TABLE_SIZE]; | |
70 | ||
71 | /** | |
72 | * __mark_empty_function - Empty probe callback | |
fb40bd78 MD |
73 | * @probe_private: probe private data |
74 | * @call_private: call site private data | |
8256e47c MD |
75 | * @fmt: format string |
76 | * @...: variable argument list | |
77 | * | |
78 | * Empty callback provided as a probe to the markers. By providing this to a | |
79 | * disabled marker, we make sure the execution flow is always valid even | |
80 | * though the function pointer change and the marker enabling are two distinct | |
81 | * operations that modifies the execution flow of preemptible code. | |
82 | */ | |
fb40bd78 MD |
83 | void __mark_empty_function(void *probe_private, void *call_private, |
84 | const char *fmt, va_list *args) | |
8256e47c MD |
85 | { |
86 | } | |
87 | EXPORT_SYMBOL_GPL(__mark_empty_function); | |
88 | ||
fb40bd78 MD |
89 | /* |
90 | * marker_probe_cb Callback that prepares the variable argument list for probes. | |
91 | * @mdata: pointer of type struct marker | |
92 | * @call_private: caller site private data | |
93 | * @fmt: format string | |
94 | * @...: Variable argument list. | |
95 | * | |
96 | * Since we do not use "typical" pointer based RCU in the 1 argument case, we | |
97 | * need to put a full smp_rmb() in this branch. This is why we do not use | |
98 | * rcu_dereference() for the pointer read. | |
99 | */ | |
100 | void marker_probe_cb(const struct marker *mdata, void *call_private, | |
101 | const char *fmt, ...) | |
102 | { | |
103 | va_list args; | |
104 | char ptype; | |
105 | ||
106 | /* | |
fd3c36f8 MD |
107 | * preempt_disable does two things : disabling preemption to make sure |
108 | * the teardown of the callbacks can be done correctly when they are in | |
109 | * modules and they insure RCU read coherency. | |
fb40bd78 MD |
110 | */ |
111 | preempt_disable(); | |
58336114 | 112 | ptype = mdata->ptype; |
fb40bd78 MD |
113 | if (likely(!ptype)) { |
114 | marker_probe_func *func; | |
115 | /* Must read the ptype before ptr. They are not data dependant, | |
116 | * so we put an explicit smp_rmb() here. */ | |
117 | smp_rmb(); | |
58336114 | 118 | func = mdata->single.func; |
fb40bd78 MD |
119 | /* Must read the ptr before private data. They are not data |
120 | * dependant, so we put an explicit smp_rmb() here. */ | |
121 | smp_rmb(); | |
122 | va_start(args, fmt); | |
123 | func(mdata->single.probe_private, call_private, fmt, &args); | |
124 | va_end(args); | |
125 | } else { | |
126 | struct marker_probe_closure *multi; | |
127 | int i; | |
128 | /* | |
129 | * multi points to an array, therefore accessing the array | |
130 | * depends on reading multi. However, even in this case, | |
131 | * we must insure that the pointer is read _before_ the array | |
132 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
133 | * in the fast path, so put the explicit barrier here. | |
134 | */ | |
135 | smp_read_barrier_depends(); | |
58336114 | 136 | multi = mdata->multi; |
fb40bd78 MD |
137 | for (i = 0; multi[i].func; i++) { |
138 | va_start(args, fmt); | |
139 | multi[i].func(multi[i].probe_private, call_private, fmt, | |
140 | &args); | |
141 | va_end(args); | |
142 | } | |
143 | } | |
144 | preempt_enable(); | |
145 | } | |
146 | EXPORT_SYMBOL_GPL(marker_probe_cb); | |
147 | ||
148 | /* | |
149 | * marker_probe_cb Callback that does not prepare the variable argument list. | |
150 | * @mdata: pointer of type struct marker | |
151 | * @call_private: caller site private data | |
152 | * @fmt: format string | |
153 | * @...: Variable argument list. | |
154 | * | |
155 | * Should be connected to markers "MARK_NOARGS". | |
156 | */ | |
157 | void marker_probe_cb_noarg(const struct marker *mdata, | |
158 | void *call_private, const char *fmt, ...) | |
159 | { | |
160 | va_list args; /* not initialized */ | |
161 | char ptype; | |
162 | ||
163 | preempt_disable(); | |
58336114 | 164 | ptype = mdata->ptype; |
fb40bd78 MD |
165 | if (likely(!ptype)) { |
166 | marker_probe_func *func; | |
167 | /* Must read the ptype before ptr. They are not data dependant, | |
168 | * so we put an explicit smp_rmb() here. */ | |
169 | smp_rmb(); | |
58336114 | 170 | func = mdata->single.func; |
fb40bd78 MD |
171 | /* Must read the ptr before private data. They are not data |
172 | * dependant, so we put an explicit smp_rmb() here. */ | |
173 | smp_rmb(); | |
174 | func(mdata->single.probe_private, call_private, fmt, &args); | |
175 | } else { | |
176 | struct marker_probe_closure *multi; | |
177 | int i; | |
178 | /* | |
179 | * multi points to an array, therefore accessing the array | |
180 | * depends on reading multi. However, even in this case, | |
181 | * we must insure that the pointer is read _before_ the array | |
182 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
183 | * in the fast path, so put the explicit barrier here. | |
184 | */ | |
185 | smp_read_barrier_depends(); | |
58336114 | 186 | multi = mdata->multi; |
fb40bd78 MD |
187 | for (i = 0; multi[i].func; i++) |
188 | multi[i].func(multi[i].probe_private, call_private, fmt, | |
189 | &args); | |
190 | } | |
191 | preempt_enable(); | |
192 | } | |
193 | EXPORT_SYMBOL_GPL(marker_probe_cb_noarg); | |
194 | ||
195 | static void free_old_closure(struct rcu_head *head) | |
196 | { | |
197 | struct marker_entry *entry = container_of(head, | |
198 | struct marker_entry, rcu); | |
199 | kfree(entry->oldptr); | |
200 | /* Make sure we free the data before setting the pending flag to 0 */ | |
201 | smp_wmb(); | |
202 | entry->rcu_pending = 0; | |
203 | } | |
204 | ||
205 | static void debug_print_probes(struct marker_entry *entry) | |
206 | { | |
207 | int i; | |
208 | ||
209 | if (!marker_debug) | |
210 | return; | |
211 | ||
212 | if (!entry->ptype) { | |
213 | printk(KERN_DEBUG "Single probe : %p %p\n", | |
214 | entry->single.func, | |
215 | entry->single.probe_private); | |
216 | } else { | |
217 | for (i = 0; entry->multi[i].func; i++) | |
218 | printk(KERN_DEBUG "Multi probe %d : %p %p\n", i, | |
219 | entry->multi[i].func, | |
220 | entry->multi[i].probe_private); | |
221 | } | |
222 | } | |
223 | ||
224 | static struct marker_probe_closure * | |
225 | marker_entry_add_probe(struct marker_entry *entry, | |
226 | marker_probe_func *probe, void *probe_private) | |
227 | { | |
228 | int nr_probes = 0; | |
229 | struct marker_probe_closure *old, *new; | |
230 | ||
231 | WARN_ON(!probe); | |
232 | ||
233 | debug_print_probes(entry); | |
234 | old = entry->multi; | |
235 | if (!entry->ptype) { | |
236 | if (entry->single.func == probe && | |
237 | entry->single.probe_private == probe_private) | |
238 | return ERR_PTR(-EBUSY); | |
239 | if (entry->single.func == __mark_empty_function) { | |
240 | /* 0 -> 1 probes */ | |
241 | entry->single.func = probe; | |
242 | entry->single.probe_private = probe_private; | |
243 | entry->refcount = 1; | |
244 | entry->ptype = 0; | |
245 | debug_print_probes(entry); | |
246 | return NULL; | |
247 | } else { | |
248 | /* 1 -> 2 probes */ | |
249 | nr_probes = 1; | |
250 | old = NULL; | |
251 | } | |
252 | } else { | |
253 | /* (N -> N+1), (N != 0, 1) probes */ | |
254 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) | |
255 | if (old[nr_probes].func == probe | |
256 | && old[nr_probes].probe_private | |
257 | == probe_private) | |
258 | return ERR_PTR(-EBUSY); | |
259 | } | |
260 | /* + 2 : one for new probe, one for NULL func */ | |
261 | new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure), | |
262 | GFP_KERNEL); | |
263 | if (new == NULL) | |
264 | return ERR_PTR(-ENOMEM); | |
265 | if (!old) | |
266 | new[0] = entry->single; | |
267 | else | |
268 | memcpy(new, old, | |
269 | nr_probes * sizeof(struct marker_probe_closure)); | |
270 | new[nr_probes].func = probe; | |
271 | new[nr_probes].probe_private = probe_private; | |
272 | entry->refcount = nr_probes + 1; | |
273 | entry->multi = new; | |
274 | entry->ptype = 1; | |
275 | debug_print_probes(entry); | |
276 | return old; | |
277 | } | |
278 | ||
279 | static struct marker_probe_closure * | |
280 | marker_entry_remove_probe(struct marker_entry *entry, | |
281 | marker_probe_func *probe, void *probe_private) | |
282 | { | |
283 | int nr_probes = 0, nr_del = 0, i; | |
284 | struct marker_probe_closure *old, *new; | |
285 | ||
286 | old = entry->multi; | |
287 | ||
288 | debug_print_probes(entry); | |
289 | if (!entry->ptype) { | |
290 | /* 0 -> N is an error */ | |
291 | WARN_ON(entry->single.func == __mark_empty_function); | |
292 | /* 1 -> 0 probes */ | |
293 | WARN_ON(probe && entry->single.func != probe); | |
294 | WARN_ON(entry->single.probe_private != probe_private); | |
295 | entry->single.func = __mark_empty_function; | |
296 | entry->refcount = 0; | |
297 | entry->ptype = 0; | |
298 | debug_print_probes(entry); | |
299 | return NULL; | |
300 | } else { | |
301 | /* (N -> M), (N > 1, M >= 0) probes */ | |
302 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) { | |
303 | if ((!probe || old[nr_probes].func == probe) | |
304 | && old[nr_probes].probe_private | |
305 | == probe_private) | |
306 | nr_del++; | |
307 | } | |
308 | } | |
309 | ||
310 | if (nr_probes - nr_del == 0) { | |
311 | /* N -> 0, (N > 1) */ | |
312 | entry->single.func = __mark_empty_function; | |
313 | entry->refcount = 0; | |
314 | entry->ptype = 0; | |
315 | } else if (nr_probes - nr_del == 1) { | |
316 | /* N -> 1, (N > 1) */ | |
317 | for (i = 0; old[i].func; i++) | |
318 | if ((probe && old[i].func != probe) || | |
319 | old[i].probe_private != probe_private) | |
320 | entry->single = old[i]; | |
321 | entry->refcount = 1; | |
322 | entry->ptype = 0; | |
323 | } else { | |
324 | int j = 0; | |
325 | /* N -> M, (N > 1, M > 1) */ | |
326 | /* + 1 for NULL */ | |
327 | new = kzalloc((nr_probes - nr_del + 1) | |
328 | * sizeof(struct marker_probe_closure), GFP_KERNEL); | |
329 | if (new == NULL) | |
330 | return ERR_PTR(-ENOMEM); | |
331 | for (i = 0; old[i].func; i++) | |
332 | if ((probe && old[i].func != probe) || | |
333 | old[i].probe_private != probe_private) | |
334 | new[j++] = old[i]; | |
335 | entry->refcount = nr_probes - nr_del; | |
336 | entry->ptype = 1; | |
337 | entry->multi = new; | |
338 | } | |
339 | debug_print_probes(entry); | |
340 | return old; | |
341 | } | |
342 | ||
8256e47c MD |
343 | /* |
344 | * Get marker if the marker is present in the marker hash table. | |
345 | * Must be called with markers_mutex held. | |
346 | * Returns NULL if not present. | |
347 | */ | |
348 | static struct marker_entry *get_marker(const char *name) | |
349 | { | |
350 | struct hlist_head *head; | |
351 | struct hlist_node *node; | |
352 | struct marker_entry *e; | |
353 | u32 hash = jhash(name, strlen(name), 0); | |
354 | ||
355 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
356 | hlist_for_each_entry(e, node, head, hlist) { | |
357 | if (!strcmp(name, e->name)) | |
358 | return e; | |
359 | } | |
360 | return NULL; | |
361 | } | |
362 | ||
363 | /* | |
364 | * Add the marker to the marker hash table. Must be called with markers_mutex | |
365 | * held. | |
366 | */ | |
fb40bd78 | 367 | static struct marker_entry *add_marker(const char *name, const char *format) |
8256e47c MD |
368 | { |
369 | struct hlist_head *head; | |
370 | struct hlist_node *node; | |
371 | struct marker_entry *e; | |
372 | size_t name_len = strlen(name) + 1; | |
373 | size_t format_len = 0; | |
374 | u32 hash = jhash(name, name_len-1, 0); | |
375 | ||
376 | if (format) | |
377 | format_len = strlen(format) + 1; | |
378 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
379 | hlist_for_each_entry(e, node, head, hlist) { | |
380 | if (!strcmp(name, e->name)) { | |
381 | printk(KERN_NOTICE | |
fb40bd78 MD |
382 | "Marker %s busy\n", name); |
383 | return ERR_PTR(-EBUSY); /* Already there */ | |
8256e47c MD |
384 | } |
385 | } | |
386 | /* | |
387 | * Using kmalloc here to allocate a variable length element. Could | |
388 | * cause some memory fragmentation if overused. | |
389 | */ | |
390 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, | |
391 | GFP_KERNEL); | |
392 | if (!e) | |
fb40bd78 | 393 | return ERR_PTR(-ENOMEM); |
8256e47c MD |
394 | memcpy(&e->name[0], name, name_len); |
395 | if (format) { | |
396 | e->format = &e->name[name_len]; | |
397 | memcpy(e->format, format, format_len); | |
fb40bd78 MD |
398 | if (strcmp(e->format, MARK_NOARGS) == 0) |
399 | e->call = marker_probe_cb_noarg; | |
400 | else | |
401 | e->call = marker_probe_cb; | |
8256e47c MD |
402 | trace_mark(core_marker_format, "name %s format %s", |
403 | e->name, e->format); | |
fb40bd78 | 404 | } else { |
8256e47c | 405 | e->format = NULL; |
fb40bd78 MD |
406 | e->call = marker_probe_cb; |
407 | } | |
408 | e->single.func = __mark_empty_function; | |
409 | e->single.probe_private = NULL; | |
410 | e->multi = NULL; | |
411 | e->ptype = 0; | |
8256e47c | 412 | e->refcount = 0; |
fb40bd78 | 413 | e->rcu_pending = 0; |
8256e47c | 414 | hlist_add_head(&e->hlist, head); |
fb40bd78 | 415 | return e; |
8256e47c MD |
416 | } |
417 | ||
418 | /* | |
419 | * Remove the marker from the marker hash table. Must be called with mutex_lock | |
420 | * held. | |
421 | */ | |
fb40bd78 | 422 | static int remove_marker(const char *name) |
8256e47c MD |
423 | { |
424 | struct hlist_head *head; | |
425 | struct hlist_node *node; | |
426 | struct marker_entry *e; | |
427 | int found = 0; | |
428 | size_t len = strlen(name) + 1; | |
8256e47c MD |
429 | u32 hash = jhash(name, len-1, 0); |
430 | ||
431 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
432 | hlist_for_each_entry(e, node, head, hlist) { | |
433 | if (!strcmp(name, e->name)) { | |
434 | found = 1; | |
435 | break; | |
436 | } | |
437 | } | |
fb40bd78 MD |
438 | if (!found) |
439 | return -ENOENT; | |
440 | if (e->single.func != __mark_empty_function) | |
441 | return -EBUSY; | |
442 | hlist_del(&e->hlist); | |
443 | /* Make sure the call_rcu has been executed */ | |
444 | if (e->rcu_pending) | |
445 | rcu_barrier(); | |
446 | kfree(e); | |
447 | return 0; | |
8256e47c MD |
448 | } |
449 | ||
450 | /* | |
451 | * Set the mark_entry format to the format found in the element. | |
452 | */ | |
453 | static int marker_set_format(struct marker_entry **entry, const char *format) | |
454 | { | |
455 | struct marker_entry *e; | |
456 | size_t name_len = strlen((*entry)->name) + 1; | |
457 | size_t format_len = strlen(format) + 1; | |
458 | ||
fb40bd78 | 459 | |
8256e47c MD |
460 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, |
461 | GFP_KERNEL); | |
462 | if (!e) | |
463 | return -ENOMEM; | |
464 | memcpy(&e->name[0], (*entry)->name, name_len); | |
465 | e->format = &e->name[name_len]; | |
466 | memcpy(e->format, format, format_len); | |
fb40bd78 MD |
467 | if (strcmp(e->format, MARK_NOARGS) == 0) |
468 | e->call = marker_probe_cb_noarg; | |
469 | else | |
470 | e->call = marker_probe_cb; | |
471 | e->single = (*entry)->single; | |
472 | e->multi = (*entry)->multi; | |
473 | e->ptype = (*entry)->ptype; | |
8256e47c | 474 | e->refcount = (*entry)->refcount; |
fb40bd78 | 475 | e->rcu_pending = 0; |
8256e47c MD |
476 | hlist_add_before(&e->hlist, &(*entry)->hlist); |
477 | hlist_del(&(*entry)->hlist); | |
fb40bd78 MD |
478 | /* Make sure the call_rcu has been executed */ |
479 | if ((*entry)->rcu_pending) | |
480 | rcu_barrier(); | |
8256e47c MD |
481 | kfree(*entry); |
482 | *entry = e; | |
483 | trace_mark(core_marker_format, "name %s format %s", | |
484 | e->name, e->format); | |
485 | return 0; | |
486 | } | |
487 | ||
488 | /* | |
489 | * Sets the probe callback corresponding to one marker. | |
490 | */ | |
fb40bd78 MD |
491 | static int set_marker(struct marker_entry **entry, struct marker *elem, |
492 | int active) | |
8256e47c MD |
493 | { |
494 | int ret; | |
495 | WARN_ON(strcmp((*entry)->name, elem->name) != 0); | |
496 | ||
497 | if ((*entry)->format) { | |
498 | if (strcmp((*entry)->format, elem->format) != 0) { | |
499 | printk(KERN_NOTICE | |
500 | "Format mismatch for probe %s " | |
501 | "(%s), marker (%s)\n", | |
502 | (*entry)->name, | |
503 | (*entry)->format, | |
504 | elem->format); | |
505 | return -EPERM; | |
506 | } | |
507 | } else { | |
508 | ret = marker_set_format(entry, elem->format); | |
509 | if (ret) | |
510 | return ret; | |
511 | } | |
fb40bd78 MD |
512 | |
513 | /* | |
514 | * probe_cb setup (statically known) is done here. It is | |
515 | * asynchronous with the rest of execution, therefore we only | |
516 | * pass from a "safe" callback (with argument) to an "unsafe" | |
517 | * callback (does not set arguments). | |
518 | */ | |
519 | elem->call = (*entry)->call; | |
520 | /* | |
521 | * Sanity check : | |
522 | * We only update the single probe private data when the ptr is | |
523 | * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1) | |
524 | */ | |
525 | WARN_ON(elem->single.func != __mark_empty_function | |
526 | && elem->single.probe_private | |
527 | != (*entry)->single.probe_private && | |
528 | !elem->ptype); | |
529 | elem->single.probe_private = (*entry)->single.probe_private; | |
530 | /* | |
531 | * Make sure the private data is valid when we update the | |
532 | * single probe ptr. | |
533 | */ | |
534 | smp_wmb(); | |
535 | elem->single.func = (*entry)->single.func; | |
536 | /* | |
537 | * We also make sure that the new probe callbacks array is consistent | |
538 | * before setting a pointer to it. | |
539 | */ | |
540 | rcu_assign_pointer(elem->multi, (*entry)->multi); | |
541 | /* | |
542 | * Update the function or multi probe array pointer before setting the | |
543 | * ptype. | |
544 | */ | |
545 | smp_wmb(); | |
546 | elem->ptype = (*entry)->ptype; | |
547 | elem->state = active; | |
548 | ||
8256e47c MD |
549 | return 0; |
550 | } | |
551 | ||
552 | /* | |
553 | * Disable a marker and its probe callback. | |
fd3c36f8 MD |
554 | * Note: only waiting an RCU period after setting elem->call to the empty |
555 | * function insures that the original callback is not used anymore. This insured | |
556 | * by preempt_disable around the call site. | |
8256e47c MD |
557 | */ |
558 | static void disable_marker(struct marker *elem) | |
559 | { | |
fb40bd78 | 560 | /* leave "call" as is. It is known statically. */ |
8256e47c | 561 | elem->state = 0; |
fb40bd78 MD |
562 | elem->single.func = __mark_empty_function; |
563 | /* Update the function before setting the ptype */ | |
564 | smp_wmb(); | |
565 | elem->ptype = 0; /* single probe */ | |
8256e47c MD |
566 | /* |
567 | * Leave the private data and id there, because removal is racy and | |
fd3c36f8 MD |
568 | * should be done only after an RCU period. These are never used until |
569 | * the next initialization anyway. | |
8256e47c MD |
570 | */ |
571 | } | |
572 | ||
573 | /** | |
574 | * marker_update_probe_range - Update a probe range | |
575 | * @begin: beginning of the range | |
576 | * @end: end of the range | |
8256e47c MD |
577 | * |
578 | * Updates the probe callback corresponding to a range of markers. | |
8256e47c MD |
579 | */ |
580 | void marker_update_probe_range(struct marker *begin, | |
fb40bd78 | 581 | struct marker *end) |
8256e47c MD |
582 | { |
583 | struct marker *iter; | |
584 | struct marker_entry *mark_entry; | |
585 | ||
314de8a9 | 586 | mutex_lock(&markers_mutex); |
8256e47c MD |
587 | for (iter = begin; iter < end; iter++) { |
588 | mark_entry = get_marker(iter->name); | |
fb40bd78 MD |
589 | if (mark_entry) { |
590 | set_marker(&mark_entry, iter, | |
591 | !!mark_entry->refcount); | |
8256e47c MD |
592 | /* |
593 | * ignore error, continue | |
594 | */ | |
8256e47c MD |
595 | } else { |
596 | disable_marker(iter); | |
597 | } | |
598 | } | |
314de8a9 | 599 | mutex_unlock(&markers_mutex); |
8256e47c MD |
600 | } |
601 | ||
602 | /* | |
603 | * Update probes, removing the faulty probes. | |
fb40bd78 MD |
604 | * |
605 | * Internal callback only changed before the first probe is connected to it. | |
606 | * Single probe private data can only be changed on 0 -> 1 and 2 -> 1 | |
607 | * transitions. All other transitions will leave the old private data valid. | |
608 | * This makes the non-atomicity of the callback/private data updates valid. | |
609 | * | |
610 | * "special case" updates : | |
611 | * 0 -> 1 callback | |
612 | * 1 -> 0 callback | |
613 | * 1 -> 2 callbacks | |
614 | * 2 -> 1 callbacks | |
615 | * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates. | |
616 | * Site effect : marker_set_format may delete the marker entry (creating a | |
617 | * replacement). | |
8256e47c | 618 | */ |
fb40bd78 | 619 | static void marker_update_probes(void) |
8256e47c | 620 | { |
8256e47c | 621 | /* Core kernel markers */ |
fb40bd78 | 622 | marker_update_probe_range(__start___markers, __stop___markers); |
8256e47c | 623 | /* Markers in modules. */ |
fb40bd78 | 624 | module_update_markers(); |
8256e47c MD |
625 | } |
626 | ||
627 | /** | |
628 | * marker_probe_register - Connect a probe to a marker | |
629 | * @name: marker name | |
630 | * @format: format string | |
631 | * @probe: probe handler | |
fb40bd78 | 632 | * @probe_private: probe private data |
8256e47c MD |
633 | * |
634 | * private data must be a valid allocated memory address, or NULL. | |
635 | * Returns 0 if ok, error value on error. | |
fb40bd78 | 636 | * The probe address must at least be aligned on the architecture pointer size. |
8256e47c MD |
637 | */ |
638 | int marker_probe_register(const char *name, const char *format, | |
fb40bd78 | 639 | marker_probe_func *probe, void *probe_private) |
8256e47c MD |
640 | { |
641 | struct marker_entry *entry; | |
314de8a9 | 642 | int ret = 0; |
fb40bd78 | 643 | struct marker_probe_closure *old; |
8256e47c MD |
644 | |
645 | mutex_lock(&markers_mutex); | |
646 | entry = get_marker(name); | |
fb40bd78 MD |
647 | if (!entry) { |
648 | entry = add_marker(name, format); | |
649 | if (IS_ERR(entry)) { | |
650 | ret = PTR_ERR(entry); | |
651 | goto end; | |
652 | } | |
8256e47c | 653 | } |
fb40bd78 MD |
654 | /* |
655 | * If we detect that a call_rcu is pending for this marker, | |
656 | * make sure it's executed now. | |
657 | */ | |
658 | if (entry->rcu_pending) | |
659 | rcu_barrier(); | |
660 | old = marker_entry_add_probe(entry, probe, probe_private); | |
661 | if (IS_ERR(old)) { | |
662 | ret = PTR_ERR(old); | |
8256e47c | 663 | goto end; |
fb40bd78 | 664 | } |
314de8a9 | 665 | mutex_unlock(&markers_mutex); |
fb40bd78 MD |
666 | marker_update_probes(); /* may update entry */ |
667 | mutex_lock(&markers_mutex); | |
668 | entry = get_marker(name); | |
669 | WARN_ON(!entry); | |
670 | entry->oldptr = old; | |
671 | entry->rcu_pending = 1; | |
672 | /* write rcu_pending before calling the RCU callback */ | |
673 | smp_wmb(); | |
6496968e MD |
674 | #ifdef CONFIG_PREEMPT_RCU |
675 | synchronize_sched(); /* Until we have the call_rcu_sched() */ | |
676 | #endif | |
fb40bd78 | 677 | call_rcu(&entry->rcu, free_old_closure); |
8256e47c MD |
678 | end: |
679 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
680 | return ret; |
681 | } | |
682 | EXPORT_SYMBOL_GPL(marker_probe_register); | |
683 | ||
684 | /** | |
685 | * marker_probe_unregister - Disconnect a probe from a marker | |
686 | * @name: marker name | |
fb40bd78 MD |
687 | * @probe: probe function pointer |
688 | * @probe_private: probe private data | |
8256e47c MD |
689 | * |
690 | * Returns the private data given to marker_probe_register, or an ERR_PTR(). | |
fb40bd78 MD |
691 | * We do not need to call a synchronize_sched to make sure the probes have |
692 | * finished running before doing a module unload, because the module unload | |
693 | * itself uses stop_machine(), which insures that every preempt disabled section | |
694 | * have finished. | |
8256e47c | 695 | */ |
fb40bd78 MD |
696 | int marker_probe_unregister(const char *name, |
697 | marker_probe_func *probe, void *probe_private) | |
8256e47c | 698 | { |
8256e47c | 699 | struct marker_entry *entry; |
fb40bd78 | 700 | struct marker_probe_closure *old; |
544adb41 | 701 | int ret = -ENOENT; |
8256e47c MD |
702 | |
703 | mutex_lock(&markers_mutex); | |
704 | entry = get_marker(name); | |
544adb41 | 705 | if (!entry) |
8256e47c | 706 | goto end; |
fb40bd78 MD |
707 | if (entry->rcu_pending) |
708 | rcu_barrier(); | |
709 | old = marker_entry_remove_probe(entry, probe, probe_private); | |
314de8a9 | 710 | mutex_unlock(&markers_mutex); |
fb40bd78 MD |
711 | marker_update_probes(); /* may update entry */ |
712 | mutex_lock(&markers_mutex); | |
713 | entry = get_marker(name); | |
544adb41 JJ |
714 | if (!entry) |
715 | goto end; | |
fb40bd78 MD |
716 | entry->oldptr = old; |
717 | entry->rcu_pending = 1; | |
718 | /* write rcu_pending before calling the RCU callback */ | |
719 | smp_wmb(); | |
6496968e MD |
720 | #ifdef CONFIG_PREEMPT_RCU |
721 | synchronize_sched(); /* Until we have the call_rcu_sched() */ | |
722 | #endif | |
fb40bd78 MD |
723 | call_rcu(&entry->rcu, free_old_closure); |
724 | remove_marker(name); /* Ignore busy error message */ | |
544adb41 | 725 | ret = 0; |
8256e47c MD |
726 | end: |
727 | mutex_unlock(&markers_mutex); | |
fb40bd78 | 728 | return ret; |
8256e47c MD |
729 | } |
730 | EXPORT_SYMBOL_GPL(marker_probe_unregister); | |
731 | ||
fb40bd78 MD |
732 | static struct marker_entry * |
733 | get_marker_from_private_data(marker_probe_func *probe, void *probe_private) | |
8256e47c | 734 | { |
8256e47c | 735 | struct marker_entry *entry; |
8256e47c | 736 | unsigned int i; |
fb40bd78 MD |
737 | struct hlist_head *head; |
738 | struct hlist_node *node; | |
8256e47c | 739 | |
8256e47c MD |
740 | for (i = 0; i < MARKER_TABLE_SIZE; i++) { |
741 | head = &marker_table[i]; | |
742 | hlist_for_each_entry(entry, node, head, hlist) { | |
fb40bd78 MD |
743 | if (!entry->ptype) { |
744 | if (entry->single.func == probe | |
745 | && entry->single.probe_private | |
746 | == probe_private) | |
747 | return entry; | |
748 | } else { | |
749 | struct marker_probe_closure *closure; | |
750 | closure = entry->multi; | |
751 | for (i = 0; closure[i].func; i++) { | |
752 | if (closure[i].func == probe && | |
753 | closure[i].probe_private | |
754 | == probe_private) | |
755 | return entry; | |
756 | } | |
8256e47c MD |
757 | } |
758 | } | |
759 | } | |
fb40bd78 | 760 | return NULL; |
8256e47c | 761 | } |
8256e47c MD |
762 | |
763 | /** | |
fb40bd78 MD |
764 | * marker_probe_unregister_private_data - Disconnect a probe from a marker |
765 | * @probe: probe function | |
766 | * @probe_private: probe private data | |
8256e47c | 767 | * |
fb40bd78 MD |
768 | * Unregister a probe by providing the registered private data. |
769 | * Only removes the first marker found in hash table. | |
770 | * Return 0 on success or error value. | |
771 | * We do not need to call a synchronize_sched to make sure the probes have | |
772 | * finished running before doing a module unload, because the module unload | |
773 | * itself uses stop_machine(), which insures that every preempt disabled section | |
774 | * have finished. | |
8256e47c | 775 | */ |
fb40bd78 MD |
776 | int marker_probe_unregister_private_data(marker_probe_func *probe, |
777 | void *probe_private) | |
8256e47c MD |
778 | { |
779 | struct marker_entry *entry; | |
314de8a9 | 780 | int ret = 0; |
fb40bd78 | 781 | struct marker_probe_closure *old; |
8256e47c MD |
782 | |
783 | mutex_lock(&markers_mutex); | |
fb40bd78 | 784 | entry = get_marker_from_private_data(probe, probe_private); |
8256e47c MD |
785 | if (!entry) { |
786 | ret = -ENOENT; | |
787 | goto end; | |
788 | } | |
fb40bd78 MD |
789 | if (entry->rcu_pending) |
790 | rcu_barrier(); | |
791 | old = marker_entry_remove_probe(entry, NULL, probe_private); | |
8256e47c | 792 | mutex_unlock(&markers_mutex); |
fb40bd78 | 793 | marker_update_probes(); /* may update entry */ |
8256e47c | 794 | mutex_lock(&markers_mutex); |
fb40bd78 MD |
795 | entry = get_marker_from_private_data(probe, probe_private); |
796 | WARN_ON(!entry); | |
797 | entry->oldptr = old; | |
798 | entry->rcu_pending = 1; | |
799 | /* write rcu_pending before calling the RCU callback */ | |
800 | smp_wmb(); | |
6496968e MD |
801 | #ifdef CONFIG_PREEMPT_RCU |
802 | synchronize_sched(); /* Until we have the call_rcu_sched() */ | |
803 | #endif | |
fb40bd78 MD |
804 | call_rcu(&entry->rcu, free_old_closure); |
805 | remove_marker(entry->name); /* Ignore busy error message */ | |
8256e47c MD |
806 | end: |
807 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
808 | return ret; |
809 | } | |
fb40bd78 | 810 | EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); |
8256e47c MD |
811 | |
812 | /** | |
813 | * marker_get_private_data - Get a marker's probe private data | |
814 | * @name: marker name | |
fb40bd78 MD |
815 | * @probe: probe to match |
816 | * @num: get the nth matching probe's private data | |
8256e47c | 817 | * |
fb40bd78 MD |
818 | * Returns the nth private data pointer (starting from 0) matching, or an |
819 | * ERR_PTR. | |
8256e47c MD |
820 | * Returns the private data pointer, or an ERR_PTR. |
821 | * The private data pointer should _only_ be dereferenced if the caller is the | |
822 | * owner of the data, or its content could vanish. This is mostly used to | |
823 | * confirm that a caller is the owner of a registered probe. | |
824 | */ | |
fb40bd78 MD |
825 | void *marker_get_private_data(const char *name, marker_probe_func *probe, |
826 | int num) | |
8256e47c MD |
827 | { |
828 | struct hlist_head *head; | |
829 | struct hlist_node *node; | |
830 | struct marker_entry *e; | |
831 | size_t name_len = strlen(name) + 1; | |
832 | u32 hash = jhash(name, name_len-1, 0); | |
fb40bd78 | 833 | int i; |
8256e47c MD |
834 | |
835 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
836 | hlist_for_each_entry(e, node, head, hlist) { | |
837 | if (!strcmp(name, e->name)) { | |
fb40bd78 MD |
838 | if (!e->ptype) { |
839 | if (num == 0 && e->single.func == probe) | |
840 | return e->single.probe_private; | |
841 | else | |
842 | break; | |
843 | } else { | |
844 | struct marker_probe_closure *closure; | |
845 | int match = 0; | |
846 | closure = e->multi; | |
847 | for (i = 0; closure[i].func; i++) { | |
848 | if (closure[i].func != probe) | |
849 | continue; | |
850 | if (match++ == num) | |
851 | return closure[i].probe_private; | |
852 | } | |
853 | } | |
8256e47c MD |
854 | } |
855 | } | |
856 | return ERR_PTR(-ENOENT); | |
857 | } | |
858 | EXPORT_SYMBOL_GPL(marker_get_private_data); |