Commit | Line | Data |
---|---|---|
8256e47c MD |
1 | /* |
2 | * Copyright (C) 2007 Mathieu Desnoyers | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License as published by | |
6 | * the Free Software Foundation; either version 2 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | */ | |
18 | #include <linux/module.h> | |
19 | #include <linux/mutex.h> | |
20 | #include <linux/types.h> | |
21 | #include <linux/jhash.h> | |
22 | #include <linux/list.h> | |
23 | #include <linux/rcupdate.h> | |
24 | #include <linux/marker.h> | |
25 | #include <linux/err.h> | |
1aeb272c | 26 | #include <linux/slab.h> |
8256e47c MD |
27 | |
28 | extern struct marker __start___markers[]; | |
29 | extern struct marker __stop___markers[]; | |
30 | ||
fb40bd78 | 31 | /* Set to 1 to enable marker debug output */ |
ab883af5 | 32 | static const int marker_debug; |
fb40bd78 | 33 | |
8256e47c | 34 | /* |
314de8a9 | 35 | * markers_mutex nests inside module_mutex. Markers mutex protects the builtin |
fb40bd78 | 36 | * and module markers and the hash table. |
8256e47c MD |
37 | */ |
38 | static DEFINE_MUTEX(markers_mutex); | |
39 | ||
8256e47c MD |
40 | /* |
41 | * Marker hash table, containing the active markers. | |
42 | * Protected by module_mutex. | |
43 | */ | |
44 | #define MARKER_HASH_BITS 6 | |
45 | #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) | |
46 | ||
fb40bd78 MD |
47 | /* |
48 | * Note about RCU : | |
49 | * It is used to make sure every handler has finished using its private data | |
50 | * between two consecutive operation (add or remove) on a given marker. It is | |
51 | * also used to delay the free of multiple probes array until a quiescent state | |
52 | * is reached. | |
53 | * marker entries modifications are protected by the markers_mutex. | |
54 | */ | |
8256e47c MD |
55 | struct marker_entry { |
56 | struct hlist_node hlist; | |
57 | char *format; | |
dc102a8f MD |
58 | /* Probe wrapper */ |
59 | void (*call)(const struct marker *mdata, void *call_private, ...); | |
fb40bd78 MD |
60 | struct marker_probe_closure single; |
61 | struct marker_probe_closure *multi; | |
8256e47c | 62 | int refcount; /* Number of times armed. 0 if disarmed. */ |
de4fc64f | 63 | unsigned char ptype:1; |
8256e47c MD |
64 | char name[0]; /* Contains name'\0'format'\0' */ |
65 | }; | |
66 | ||
67 | static struct hlist_head marker_table[MARKER_TABLE_SIZE]; | |
68 | ||
69 | /** | |
70 | * __mark_empty_function - Empty probe callback | |
fb40bd78 MD |
71 | * @probe_private: probe private data |
72 | * @call_private: call site private data | |
8256e47c MD |
73 | * @fmt: format string |
74 | * @...: variable argument list | |
75 | * | |
76 | * Empty callback provided as a probe to the markers. By providing this to a | |
77 | * disabled marker, we make sure the execution flow is always valid even | |
78 | * though the function pointer change and the marker enabling are two distinct | |
79 | * operations that modifies the execution flow of preemptible code. | |
80 | */ | |
fb40bd78 MD |
81 | void __mark_empty_function(void *probe_private, void *call_private, |
82 | const char *fmt, va_list *args) | |
8256e47c MD |
83 | { |
84 | } | |
85 | EXPORT_SYMBOL_GPL(__mark_empty_function); | |
86 | ||
fb40bd78 MD |
87 | /* |
88 | * marker_probe_cb Callback that prepares the variable argument list for probes. | |
89 | * @mdata: pointer of type struct marker | |
90 | * @call_private: caller site private data | |
fb40bd78 MD |
91 | * @...: Variable argument list. |
92 | * | |
93 | * Since we do not use "typical" pointer based RCU in the 1 argument case, we | |
94 | * need to put a full smp_rmb() in this branch. This is why we do not use | |
95 | * rcu_dereference() for the pointer read. | |
96 | */ | |
dc102a8f | 97 | void marker_probe_cb(const struct marker *mdata, void *call_private, ...) |
fb40bd78 MD |
98 | { |
99 | va_list args; | |
100 | char ptype; | |
101 | ||
102 | /* | |
fd3c36f8 MD |
103 | * preempt_disable does two things : disabling preemption to make sure |
104 | * the teardown of the callbacks can be done correctly when they are in | |
105 | * modules and they insure RCU read coherency. | |
fb40bd78 MD |
106 | */ |
107 | preempt_disable(); | |
58336114 | 108 | ptype = mdata->ptype; |
fb40bd78 MD |
109 | if (likely(!ptype)) { |
110 | marker_probe_func *func; | |
111 | /* Must read the ptype before ptr. They are not data dependant, | |
112 | * so we put an explicit smp_rmb() here. */ | |
113 | smp_rmb(); | |
58336114 | 114 | func = mdata->single.func; |
fb40bd78 MD |
115 | /* Must read the ptr before private data. They are not data |
116 | * dependant, so we put an explicit smp_rmb() here. */ | |
117 | smp_rmb(); | |
dc102a8f MD |
118 | va_start(args, call_private); |
119 | func(mdata->single.probe_private, call_private, mdata->format, | |
120 | &args); | |
fb40bd78 MD |
121 | va_end(args); |
122 | } else { | |
123 | struct marker_probe_closure *multi; | |
124 | int i; | |
5def9a3a MD |
125 | /* |
126 | * Read mdata->ptype before mdata->multi. | |
127 | */ | |
128 | smp_rmb(); | |
129 | multi = mdata->multi; | |
fb40bd78 MD |
130 | /* |
131 | * multi points to an array, therefore accessing the array | |
132 | * depends on reading multi. However, even in this case, | |
133 | * we must insure that the pointer is read _before_ the array | |
134 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
135 | * in the fast path, so put the explicit barrier here. | |
136 | */ | |
137 | smp_read_barrier_depends(); | |
fb40bd78 | 138 | for (i = 0; multi[i].func; i++) { |
dc102a8f MD |
139 | va_start(args, call_private); |
140 | multi[i].func(multi[i].probe_private, call_private, | |
141 | mdata->format, &args); | |
fb40bd78 MD |
142 | va_end(args); |
143 | } | |
144 | } | |
145 | preempt_enable(); | |
146 | } | |
147 | EXPORT_SYMBOL_GPL(marker_probe_cb); | |
148 | ||
149 | /* | |
150 | * marker_probe_cb Callback that does not prepare the variable argument list. | |
151 | * @mdata: pointer of type struct marker | |
152 | * @call_private: caller site private data | |
fb40bd78 MD |
153 | * @...: Variable argument list. |
154 | * | |
155 | * Should be connected to markers "MARK_NOARGS". | |
156 | */ | |
dc102a8f | 157 | void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...) |
fb40bd78 MD |
158 | { |
159 | va_list args; /* not initialized */ | |
160 | char ptype; | |
161 | ||
162 | preempt_disable(); | |
58336114 | 163 | ptype = mdata->ptype; |
fb40bd78 MD |
164 | if (likely(!ptype)) { |
165 | marker_probe_func *func; | |
166 | /* Must read the ptype before ptr. They are not data dependant, | |
167 | * so we put an explicit smp_rmb() here. */ | |
168 | smp_rmb(); | |
58336114 | 169 | func = mdata->single.func; |
fb40bd78 MD |
170 | /* Must read the ptr before private data. They are not data |
171 | * dependant, so we put an explicit smp_rmb() here. */ | |
172 | smp_rmb(); | |
dc102a8f MD |
173 | func(mdata->single.probe_private, call_private, mdata->format, |
174 | &args); | |
fb40bd78 MD |
175 | } else { |
176 | struct marker_probe_closure *multi; | |
177 | int i; | |
5def9a3a MD |
178 | /* |
179 | * Read mdata->ptype before mdata->multi. | |
180 | */ | |
181 | smp_rmb(); | |
182 | multi = mdata->multi; | |
fb40bd78 MD |
183 | /* |
184 | * multi points to an array, therefore accessing the array | |
185 | * depends on reading multi. However, even in this case, | |
186 | * we must insure that the pointer is read _before_ the array | |
187 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
188 | * in the fast path, so put the explicit barrier here. | |
189 | */ | |
190 | smp_read_barrier_depends(); | |
fb40bd78 | 191 | for (i = 0; multi[i].func; i++) |
dc102a8f MD |
192 | multi[i].func(multi[i].probe_private, call_private, |
193 | mdata->format, &args); | |
fb40bd78 MD |
194 | } |
195 | preempt_enable(); | |
196 | } | |
197 | EXPORT_SYMBOL_GPL(marker_probe_cb_noarg); | |
198 | ||
fb40bd78 MD |
199 | static void debug_print_probes(struct marker_entry *entry) |
200 | { | |
201 | int i; | |
202 | ||
203 | if (!marker_debug) | |
204 | return; | |
205 | ||
206 | if (!entry->ptype) { | |
207 | printk(KERN_DEBUG "Single probe : %p %p\n", | |
208 | entry->single.func, | |
209 | entry->single.probe_private); | |
210 | } else { | |
211 | for (i = 0; entry->multi[i].func; i++) | |
212 | printk(KERN_DEBUG "Multi probe %d : %p %p\n", i, | |
213 | entry->multi[i].func, | |
214 | entry->multi[i].probe_private); | |
215 | } | |
216 | } | |
217 | ||
218 | static struct marker_probe_closure * | |
219 | marker_entry_add_probe(struct marker_entry *entry, | |
220 | marker_probe_func *probe, void *probe_private) | |
221 | { | |
222 | int nr_probes = 0; | |
223 | struct marker_probe_closure *old, *new; | |
224 | ||
225 | WARN_ON(!probe); | |
226 | ||
227 | debug_print_probes(entry); | |
228 | old = entry->multi; | |
229 | if (!entry->ptype) { | |
230 | if (entry->single.func == probe && | |
231 | entry->single.probe_private == probe_private) | |
232 | return ERR_PTR(-EBUSY); | |
233 | if (entry->single.func == __mark_empty_function) { | |
234 | /* 0 -> 1 probes */ | |
235 | entry->single.func = probe; | |
236 | entry->single.probe_private = probe_private; | |
237 | entry->refcount = 1; | |
238 | entry->ptype = 0; | |
239 | debug_print_probes(entry); | |
240 | return NULL; | |
241 | } else { | |
242 | /* 1 -> 2 probes */ | |
243 | nr_probes = 1; | |
244 | old = NULL; | |
245 | } | |
246 | } else { | |
247 | /* (N -> N+1), (N != 0, 1) probes */ | |
248 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) | |
249 | if (old[nr_probes].func == probe | |
250 | && old[nr_probes].probe_private | |
251 | == probe_private) | |
252 | return ERR_PTR(-EBUSY); | |
253 | } | |
254 | /* + 2 : one for new probe, one for NULL func */ | |
255 | new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure), | |
256 | GFP_KERNEL); | |
257 | if (new == NULL) | |
258 | return ERR_PTR(-ENOMEM); | |
259 | if (!old) | |
260 | new[0] = entry->single; | |
261 | else | |
262 | memcpy(new, old, | |
263 | nr_probes * sizeof(struct marker_probe_closure)); | |
264 | new[nr_probes].func = probe; | |
265 | new[nr_probes].probe_private = probe_private; | |
266 | entry->refcount = nr_probes + 1; | |
267 | entry->multi = new; | |
268 | entry->ptype = 1; | |
269 | debug_print_probes(entry); | |
270 | return old; | |
271 | } | |
272 | ||
273 | static struct marker_probe_closure * | |
274 | marker_entry_remove_probe(struct marker_entry *entry, | |
275 | marker_probe_func *probe, void *probe_private) | |
276 | { | |
277 | int nr_probes = 0, nr_del = 0, i; | |
278 | struct marker_probe_closure *old, *new; | |
279 | ||
280 | old = entry->multi; | |
281 | ||
282 | debug_print_probes(entry); | |
283 | if (!entry->ptype) { | |
284 | /* 0 -> N is an error */ | |
285 | WARN_ON(entry->single.func == __mark_empty_function); | |
286 | /* 1 -> 0 probes */ | |
287 | WARN_ON(probe && entry->single.func != probe); | |
288 | WARN_ON(entry->single.probe_private != probe_private); | |
289 | entry->single.func = __mark_empty_function; | |
290 | entry->refcount = 0; | |
291 | entry->ptype = 0; | |
292 | debug_print_probes(entry); | |
293 | return NULL; | |
294 | } else { | |
295 | /* (N -> M), (N > 1, M >= 0) probes */ | |
296 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) { | |
297 | if ((!probe || old[nr_probes].func == probe) | |
298 | && old[nr_probes].probe_private | |
299 | == probe_private) | |
300 | nr_del++; | |
301 | } | |
302 | } | |
303 | ||
304 | if (nr_probes - nr_del == 0) { | |
305 | /* N -> 0, (N > 1) */ | |
306 | entry->single.func = __mark_empty_function; | |
307 | entry->refcount = 0; | |
308 | entry->ptype = 0; | |
309 | } else if (nr_probes - nr_del == 1) { | |
310 | /* N -> 1, (N > 1) */ | |
311 | for (i = 0; old[i].func; i++) | |
312 | if ((probe && old[i].func != probe) || | |
313 | old[i].probe_private != probe_private) | |
314 | entry->single = old[i]; | |
315 | entry->refcount = 1; | |
316 | entry->ptype = 0; | |
317 | } else { | |
318 | int j = 0; | |
319 | /* N -> M, (N > 1, M > 1) */ | |
320 | /* + 1 for NULL */ | |
321 | new = kzalloc((nr_probes - nr_del + 1) | |
322 | * sizeof(struct marker_probe_closure), GFP_KERNEL); | |
323 | if (new == NULL) | |
324 | return ERR_PTR(-ENOMEM); | |
325 | for (i = 0; old[i].func; i++) | |
326 | if ((probe && old[i].func != probe) || | |
327 | old[i].probe_private != probe_private) | |
328 | new[j++] = old[i]; | |
329 | entry->refcount = nr_probes - nr_del; | |
330 | entry->ptype = 1; | |
331 | entry->multi = new; | |
332 | } | |
333 | debug_print_probes(entry); | |
334 | return old; | |
335 | } | |
336 | ||
8256e47c MD |
337 | /* |
338 | * Get marker if the marker is present in the marker hash table. | |
339 | * Must be called with markers_mutex held. | |
340 | * Returns NULL if not present. | |
341 | */ | |
342 | static struct marker_entry *get_marker(const char *name) | |
343 | { | |
344 | struct hlist_head *head; | |
345 | struct hlist_node *node; | |
346 | struct marker_entry *e; | |
347 | u32 hash = jhash(name, strlen(name), 0); | |
348 | ||
349 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
350 | hlist_for_each_entry(e, node, head, hlist) { | |
351 | if (!strcmp(name, e->name)) | |
352 | return e; | |
353 | } | |
354 | return NULL; | |
355 | } | |
356 | ||
357 | /* | |
358 | * Add the marker to the marker hash table. Must be called with markers_mutex | |
359 | * held. | |
360 | */ | |
fb40bd78 | 361 | static struct marker_entry *add_marker(const char *name, const char *format) |
8256e47c MD |
362 | { |
363 | struct hlist_head *head; | |
364 | struct hlist_node *node; | |
365 | struct marker_entry *e; | |
366 | size_t name_len = strlen(name) + 1; | |
367 | size_t format_len = 0; | |
368 | u32 hash = jhash(name, name_len-1, 0); | |
369 | ||
370 | if (format) | |
371 | format_len = strlen(format) + 1; | |
372 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
373 | hlist_for_each_entry(e, node, head, hlist) { | |
374 | if (!strcmp(name, e->name)) { | |
375 | printk(KERN_NOTICE | |
fb40bd78 MD |
376 | "Marker %s busy\n", name); |
377 | return ERR_PTR(-EBUSY); /* Already there */ | |
8256e47c MD |
378 | } |
379 | } | |
380 | /* | |
381 | * Using kmalloc here to allocate a variable length element. Could | |
382 | * cause some memory fragmentation if overused. | |
383 | */ | |
384 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, | |
385 | GFP_KERNEL); | |
386 | if (!e) | |
fb40bd78 | 387 | return ERR_PTR(-ENOMEM); |
8256e47c MD |
388 | memcpy(&e->name[0], name, name_len); |
389 | if (format) { | |
390 | e->format = &e->name[name_len]; | |
391 | memcpy(e->format, format, format_len); | |
fb40bd78 MD |
392 | if (strcmp(e->format, MARK_NOARGS) == 0) |
393 | e->call = marker_probe_cb_noarg; | |
394 | else | |
395 | e->call = marker_probe_cb; | |
8256e47c MD |
396 | trace_mark(core_marker_format, "name %s format %s", |
397 | e->name, e->format); | |
fb40bd78 | 398 | } else { |
8256e47c | 399 | e->format = NULL; |
fb40bd78 MD |
400 | e->call = marker_probe_cb; |
401 | } | |
402 | e->single.func = __mark_empty_function; | |
403 | e->single.probe_private = NULL; | |
404 | e->multi = NULL; | |
405 | e->ptype = 0; | |
8256e47c MD |
406 | e->refcount = 0; |
407 | hlist_add_head(&e->hlist, head); | |
fb40bd78 | 408 | return e; |
8256e47c MD |
409 | } |
410 | ||
411 | /* | |
412 | * Remove the marker from the marker hash table. Must be called with mutex_lock | |
413 | * held. | |
414 | */ | |
fb40bd78 | 415 | static int remove_marker(const char *name) |
8256e47c MD |
416 | { |
417 | struct hlist_head *head; | |
418 | struct hlist_node *node; | |
419 | struct marker_entry *e; | |
420 | int found = 0; | |
421 | size_t len = strlen(name) + 1; | |
8256e47c MD |
422 | u32 hash = jhash(name, len-1, 0); |
423 | ||
424 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
425 | hlist_for_each_entry(e, node, head, hlist) { | |
426 | if (!strcmp(name, e->name)) { | |
427 | found = 1; | |
428 | break; | |
429 | } | |
430 | } | |
fb40bd78 MD |
431 | if (!found) |
432 | return -ENOENT; | |
433 | if (e->single.func != __mark_empty_function) | |
434 | return -EBUSY; | |
435 | hlist_del(&e->hlist); | |
fb40bd78 MD |
436 | kfree(e); |
437 | return 0; | |
8256e47c MD |
438 | } |
439 | ||
440 | /* | |
441 | * Set the mark_entry format to the format found in the element. | |
442 | */ | |
443 | static int marker_set_format(struct marker_entry **entry, const char *format) | |
444 | { | |
445 | struct marker_entry *e; | |
446 | size_t name_len = strlen((*entry)->name) + 1; | |
447 | size_t format_len = strlen(format) + 1; | |
448 | ||
fb40bd78 | 449 | |
8256e47c MD |
450 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, |
451 | GFP_KERNEL); | |
452 | if (!e) | |
453 | return -ENOMEM; | |
454 | memcpy(&e->name[0], (*entry)->name, name_len); | |
455 | e->format = &e->name[name_len]; | |
456 | memcpy(e->format, format, format_len); | |
fb40bd78 MD |
457 | if (strcmp(e->format, MARK_NOARGS) == 0) |
458 | e->call = marker_probe_cb_noarg; | |
459 | else | |
460 | e->call = marker_probe_cb; | |
461 | e->single = (*entry)->single; | |
462 | e->multi = (*entry)->multi; | |
463 | e->ptype = (*entry)->ptype; | |
8256e47c MD |
464 | e->refcount = (*entry)->refcount; |
465 | hlist_add_before(&e->hlist, &(*entry)->hlist); | |
466 | hlist_del(&(*entry)->hlist); | |
467 | kfree(*entry); | |
468 | *entry = e; | |
469 | trace_mark(core_marker_format, "name %s format %s", | |
470 | e->name, e->format); | |
471 | return 0; | |
472 | } | |
473 | ||
474 | /* | |
475 | * Sets the probe callback corresponding to one marker. | |
476 | */ | |
fb40bd78 MD |
477 | static int set_marker(struct marker_entry **entry, struct marker *elem, |
478 | int active) | |
8256e47c MD |
479 | { |
480 | int ret; | |
481 | WARN_ON(strcmp((*entry)->name, elem->name) != 0); | |
482 | ||
483 | if ((*entry)->format) { | |
484 | if (strcmp((*entry)->format, elem->format) != 0) { | |
485 | printk(KERN_NOTICE | |
486 | "Format mismatch for probe %s " | |
487 | "(%s), marker (%s)\n", | |
488 | (*entry)->name, | |
489 | (*entry)->format, | |
490 | elem->format); | |
491 | return -EPERM; | |
492 | } | |
493 | } else { | |
494 | ret = marker_set_format(entry, elem->format); | |
495 | if (ret) | |
496 | return ret; | |
497 | } | |
fb40bd78 MD |
498 | |
499 | /* | |
500 | * probe_cb setup (statically known) is done here. It is | |
501 | * asynchronous with the rest of execution, therefore we only | |
502 | * pass from a "safe" callback (with argument) to an "unsafe" | |
503 | * callback (does not set arguments). | |
504 | */ | |
505 | elem->call = (*entry)->call; | |
506 | /* | |
507 | * Sanity check : | |
508 | * We only update the single probe private data when the ptr is | |
509 | * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1) | |
510 | */ | |
511 | WARN_ON(elem->single.func != __mark_empty_function | |
512 | && elem->single.probe_private | |
513 | != (*entry)->single.probe_private && | |
514 | !elem->ptype); | |
515 | elem->single.probe_private = (*entry)->single.probe_private; | |
516 | /* | |
517 | * Make sure the private data is valid when we update the | |
518 | * single probe ptr. | |
519 | */ | |
520 | smp_wmb(); | |
521 | elem->single.func = (*entry)->single.func; | |
522 | /* | |
523 | * We also make sure that the new probe callbacks array is consistent | |
524 | * before setting a pointer to it. | |
525 | */ | |
526 | rcu_assign_pointer(elem->multi, (*entry)->multi); | |
527 | /* | |
528 | * Update the function or multi probe array pointer before setting the | |
529 | * ptype. | |
530 | */ | |
531 | smp_wmb(); | |
532 | elem->ptype = (*entry)->ptype; | |
533 | elem->state = active; | |
534 | ||
8256e47c MD |
535 | return 0; |
536 | } | |
537 | ||
538 | /* | |
539 | * Disable a marker and its probe callback. | |
fd3c36f8 MD |
540 | * Note: only waiting an RCU period after setting elem->call to the empty |
541 | * function insures that the original callback is not used anymore. This insured | |
542 | * by preempt_disable around the call site. | |
8256e47c MD |
543 | */ |
544 | static void disable_marker(struct marker *elem) | |
545 | { | |
fb40bd78 | 546 | /* leave "call" as is. It is known statically. */ |
8256e47c | 547 | elem->state = 0; |
fb40bd78 MD |
548 | elem->single.func = __mark_empty_function; |
549 | /* Update the function before setting the ptype */ | |
550 | smp_wmb(); | |
551 | elem->ptype = 0; /* single probe */ | |
8256e47c MD |
552 | /* |
553 | * Leave the private data and id there, because removal is racy and | |
fd3c36f8 MD |
554 | * should be done only after an RCU period. These are never used until |
555 | * the next initialization anyway. | |
8256e47c MD |
556 | */ |
557 | } | |
558 | ||
559 | /** | |
560 | * marker_update_probe_range - Update a probe range | |
561 | * @begin: beginning of the range | |
562 | * @end: end of the range | |
8256e47c MD |
563 | * |
564 | * Updates the probe callback corresponding to a range of markers. | |
8256e47c MD |
565 | */ |
566 | void marker_update_probe_range(struct marker *begin, | |
fb40bd78 | 567 | struct marker *end) |
8256e47c MD |
568 | { |
569 | struct marker *iter; | |
570 | struct marker_entry *mark_entry; | |
571 | ||
314de8a9 | 572 | mutex_lock(&markers_mutex); |
8256e47c MD |
573 | for (iter = begin; iter < end; iter++) { |
574 | mark_entry = get_marker(iter->name); | |
fb40bd78 MD |
575 | if (mark_entry) { |
576 | set_marker(&mark_entry, iter, | |
577 | !!mark_entry->refcount); | |
8256e47c MD |
578 | /* |
579 | * ignore error, continue | |
580 | */ | |
8256e47c MD |
581 | } else { |
582 | disable_marker(iter); | |
583 | } | |
584 | } | |
314de8a9 | 585 | mutex_unlock(&markers_mutex); |
8256e47c MD |
586 | } |
587 | ||
588 | /* | |
589 | * Update probes, removing the faulty probes. | |
fb40bd78 MD |
590 | * |
591 | * Internal callback only changed before the first probe is connected to it. | |
592 | * Single probe private data can only be changed on 0 -> 1 and 2 -> 1 | |
593 | * transitions. All other transitions will leave the old private data valid. | |
594 | * This makes the non-atomicity of the callback/private data updates valid. | |
595 | * | |
596 | * "special case" updates : | |
597 | * 0 -> 1 callback | |
598 | * 1 -> 0 callback | |
599 | * 1 -> 2 callbacks | |
600 | * 2 -> 1 callbacks | |
601 | * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates. | |
602 | * Site effect : marker_set_format may delete the marker entry (creating a | |
603 | * replacement). | |
8256e47c | 604 | */ |
fb40bd78 | 605 | static void marker_update_probes(void) |
8256e47c | 606 | { |
8256e47c | 607 | /* Core kernel markers */ |
fb40bd78 | 608 | marker_update_probe_range(__start___markers, __stop___markers); |
8256e47c | 609 | /* Markers in modules. */ |
fb40bd78 | 610 | module_update_markers(); |
8256e47c MD |
611 | } |
612 | ||
613 | /** | |
614 | * marker_probe_register - Connect a probe to a marker | |
615 | * @name: marker name | |
616 | * @format: format string | |
617 | * @probe: probe handler | |
fb40bd78 | 618 | * @probe_private: probe private data |
8256e47c MD |
619 | * |
620 | * private data must be a valid allocated memory address, or NULL. | |
621 | * Returns 0 if ok, error value on error. | |
fb40bd78 | 622 | * The probe address must at least be aligned on the architecture pointer size. |
8256e47c MD |
623 | */ |
624 | int marker_probe_register(const char *name, const char *format, | |
fb40bd78 | 625 | marker_probe_func *probe, void *probe_private) |
8256e47c MD |
626 | { |
627 | struct marker_entry *entry; | |
314de8a9 | 628 | int ret = 0; |
fb40bd78 | 629 | struct marker_probe_closure *old; |
8256e47c MD |
630 | |
631 | mutex_lock(&markers_mutex); | |
632 | entry = get_marker(name); | |
fb40bd78 MD |
633 | if (!entry) { |
634 | entry = add_marker(name, format); | |
635 | if (IS_ERR(entry)) { | |
636 | ret = PTR_ERR(entry); | |
637 | goto end; | |
638 | } | |
8256e47c | 639 | } |
fb40bd78 MD |
640 | old = marker_entry_add_probe(entry, probe, probe_private); |
641 | if (IS_ERR(old)) { | |
642 | ret = PTR_ERR(old); | |
8256e47c | 643 | goto end; |
fb40bd78 | 644 | } |
314de8a9 | 645 | mutex_unlock(&markers_mutex); |
fb40bd78 | 646 | marker_update_probes(); /* may update entry */ |
d74185ed LJ |
647 | synchronize_sched(); |
648 | kfree(old); | |
fb40bd78 MD |
649 | mutex_lock(&markers_mutex); |
650 | entry = get_marker(name); | |
651 | WARN_ON(!entry); | |
8256e47c MD |
652 | end: |
653 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
654 | return ret; |
655 | } | |
656 | EXPORT_SYMBOL_GPL(marker_probe_register); | |
657 | ||
658 | /** | |
659 | * marker_probe_unregister - Disconnect a probe from a marker | |
660 | * @name: marker name | |
fb40bd78 MD |
661 | * @probe: probe function pointer |
662 | * @probe_private: probe private data | |
8256e47c MD |
663 | * |
664 | * Returns the private data given to marker_probe_register, or an ERR_PTR(). | |
fb40bd78 MD |
665 | * We do not need to call a synchronize_sched to make sure the probes have |
666 | * finished running before doing a module unload, because the module unload | |
667 | * itself uses stop_machine(), which insures that every preempt disabled section | |
668 | * have finished. | |
8256e47c | 669 | */ |
fb40bd78 MD |
670 | int marker_probe_unregister(const char *name, |
671 | marker_probe_func *probe, void *probe_private) | |
8256e47c | 672 | { |
8256e47c | 673 | struct marker_entry *entry; |
fb40bd78 | 674 | struct marker_probe_closure *old; |
544adb41 | 675 | int ret = -ENOENT; |
8256e47c MD |
676 | |
677 | mutex_lock(&markers_mutex); | |
678 | entry = get_marker(name); | |
544adb41 | 679 | if (!entry) |
8256e47c | 680 | goto end; |
fb40bd78 | 681 | old = marker_entry_remove_probe(entry, probe, probe_private); |
314de8a9 | 682 | mutex_unlock(&markers_mutex); |
fb40bd78 | 683 | marker_update_probes(); /* may update entry */ |
d74185ed LJ |
684 | synchronize_sched(); |
685 | kfree(old); | |
fb40bd78 MD |
686 | mutex_lock(&markers_mutex); |
687 | entry = get_marker(name); | |
544adb41 JJ |
688 | if (!entry) |
689 | goto end; | |
fb40bd78 | 690 | remove_marker(name); /* Ignore busy error message */ |
544adb41 | 691 | ret = 0; |
8256e47c MD |
692 | end: |
693 | mutex_unlock(&markers_mutex); | |
fb40bd78 | 694 | return ret; |
8256e47c MD |
695 | } |
696 | EXPORT_SYMBOL_GPL(marker_probe_unregister); | |
697 | ||
fb40bd78 MD |
698 | static struct marker_entry * |
699 | get_marker_from_private_data(marker_probe_func *probe, void *probe_private) | |
8256e47c | 700 | { |
8256e47c | 701 | struct marker_entry *entry; |
8256e47c | 702 | unsigned int i; |
fb40bd78 MD |
703 | struct hlist_head *head; |
704 | struct hlist_node *node; | |
8256e47c | 705 | |
8256e47c MD |
706 | for (i = 0; i < MARKER_TABLE_SIZE; i++) { |
707 | head = &marker_table[i]; | |
708 | hlist_for_each_entry(entry, node, head, hlist) { | |
fb40bd78 MD |
709 | if (!entry->ptype) { |
710 | if (entry->single.func == probe | |
711 | && entry->single.probe_private | |
712 | == probe_private) | |
713 | return entry; | |
714 | } else { | |
715 | struct marker_probe_closure *closure; | |
716 | closure = entry->multi; | |
717 | for (i = 0; closure[i].func; i++) { | |
718 | if (closure[i].func == probe && | |
719 | closure[i].probe_private | |
720 | == probe_private) | |
721 | return entry; | |
722 | } | |
8256e47c MD |
723 | } |
724 | } | |
725 | } | |
fb40bd78 | 726 | return NULL; |
8256e47c | 727 | } |
8256e47c MD |
728 | |
729 | /** | |
fb40bd78 MD |
730 | * marker_probe_unregister_private_data - Disconnect a probe from a marker |
731 | * @probe: probe function | |
732 | * @probe_private: probe private data | |
8256e47c | 733 | * |
fb40bd78 MD |
734 | * Unregister a probe by providing the registered private data. |
735 | * Only removes the first marker found in hash table. | |
736 | * Return 0 on success or error value. | |
737 | * We do not need to call a synchronize_sched to make sure the probes have | |
738 | * finished running before doing a module unload, because the module unload | |
739 | * itself uses stop_machine(), which insures that every preempt disabled section | |
740 | * have finished. | |
8256e47c | 741 | */ |
fb40bd78 MD |
742 | int marker_probe_unregister_private_data(marker_probe_func *probe, |
743 | void *probe_private) | |
8256e47c MD |
744 | { |
745 | struct marker_entry *entry; | |
314de8a9 | 746 | int ret = 0; |
fb40bd78 | 747 | struct marker_probe_closure *old; |
8256e47c MD |
748 | |
749 | mutex_lock(&markers_mutex); | |
fb40bd78 | 750 | entry = get_marker_from_private_data(probe, probe_private); |
8256e47c MD |
751 | if (!entry) { |
752 | ret = -ENOENT; | |
753 | goto end; | |
754 | } | |
fb40bd78 | 755 | old = marker_entry_remove_probe(entry, NULL, probe_private); |
8256e47c | 756 | mutex_unlock(&markers_mutex); |
fb40bd78 | 757 | marker_update_probes(); /* may update entry */ |
d74185ed LJ |
758 | synchronize_sched(); |
759 | kfree(old); | |
8256e47c | 760 | mutex_lock(&markers_mutex); |
fb40bd78 MD |
761 | entry = get_marker_from_private_data(probe, probe_private); |
762 | WARN_ON(!entry); | |
fb40bd78 | 763 | remove_marker(entry->name); /* Ignore busy error message */ |
8256e47c MD |
764 | end: |
765 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
766 | return ret; |
767 | } | |
fb40bd78 | 768 | EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); |
8256e47c MD |
769 | |
770 | /** | |
771 | * marker_get_private_data - Get a marker's probe private data | |
772 | * @name: marker name | |
fb40bd78 MD |
773 | * @probe: probe to match |
774 | * @num: get the nth matching probe's private data | |
8256e47c | 775 | * |
fb40bd78 MD |
776 | * Returns the nth private data pointer (starting from 0) matching, or an |
777 | * ERR_PTR. | |
8256e47c MD |
778 | * Returns the private data pointer, or an ERR_PTR. |
779 | * The private data pointer should _only_ be dereferenced if the caller is the | |
780 | * owner of the data, or its content could vanish. This is mostly used to | |
781 | * confirm that a caller is the owner of a registered probe. | |
782 | */ | |
fb40bd78 MD |
783 | void *marker_get_private_data(const char *name, marker_probe_func *probe, |
784 | int num) | |
8256e47c MD |
785 | { |
786 | struct hlist_head *head; | |
787 | struct hlist_node *node; | |
788 | struct marker_entry *e; | |
789 | size_t name_len = strlen(name) + 1; | |
790 | u32 hash = jhash(name, name_len-1, 0); | |
fb40bd78 | 791 | int i; |
8256e47c MD |
792 | |
793 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
794 | hlist_for_each_entry(e, node, head, hlist) { | |
795 | if (!strcmp(name, e->name)) { | |
fb40bd78 MD |
796 | if (!e->ptype) { |
797 | if (num == 0 && e->single.func == probe) | |
798 | return e->single.probe_private; | |
799 | else | |
800 | break; | |
801 | } else { | |
802 | struct marker_probe_closure *closure; | |
803 | int match = 0; | |
804 | closure = e->multi; | |
805 | for (i = 0; closure[i].func; i++) { | |
806 | if (closure[i].func != probe) | |
807 | continue; | |
808 | if (match++ == num) | |
809 | return closure[i].probe_private; | |
810 | } | |
811 | } | |
8256e47c MD |
812 | } |
813 | } | |
814 | return ERR_PTR(-ENOENT); | |
815 | } | |
816 | EXPORT_SYMBOL_GPL(marker_get_private_data); |