4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
6 Copyright (C) 2003-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 2003-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2003-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
10 drbd is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 2, or (at your option)
15 drbd is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
20 You should have received a copy of the GNU General Public License
21 along with drbd; see the file COPYING. If not, write to
22 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
26 #include <linux/module.h>
27 #include <linux/bitops.h>
28 #include <linux/slab.h>
29 #include <linux/string.h> /* for memset */
30 #include <linux/seq_file.h> /* for seq_printf */
31 #include <linux/lru_cache.h>
33 MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
34 "Lars Ellenberg <lars@linbit.com>");
35 MODULE_DESCRIPTION("lru_cache - Track sets of hot objects");
36 MODULE_LICENSE("GPL");
38 /* this is developers aid only.
39 * it catches concurrent access (lack of locking on the users part) */
40 #define PARANOIA_ENTRY() do { \
42 BUG_ON(!lc->nr_elements); \
43 BUG_ON(test_and_set_bit(__LC_PARANOIA, &lc->flags)); \
46 #define RETURN(x...) do { \
47 clear_bit(__LC_PARANOIA, &lc->flags); \
48 smp_mb__after_clear_bit(); return x ; } while (0)
50 /* BUG() if e is not one of the elements tracked by lc */
51 #define PARANOIA_LC_ELEMENT(lc, e) do { \
52 struct lru_cache *lc_ = (lc); \
53 struct lc_element *e_ = (e); \
54 unsigned i = e_->lc_index; \
55 BUG_ON(i >= lc_->nr_elements); \
56 BUG_ON(lc_->lc_element[i] != e_); } while (0)
59 * lc_create - prepares to track objects in an active set
60 * @name: descriptive name only used in lc_seq_printf_stats and lc_seq_dump_details
61 * @e_count: number of elements allowed to be active simultaneously
62 * @e_size: size of the tracked objects
63 * @e_off: offset to the &struct lc_element member in a tracked object
65 * Returns a pointer to a newly initialized struct lru_cache on success,
66 * or NULL on (allocation) failure.
68 struct lru_cache
*lc_create(const char *name
, struct kmem_cache
*cache
,
69 unsigned e_count
, size_t e_size
, size_t e_off
)
71 struct hlist_head
*slot
= NULL
;
72 struct lc_element
**element
= NULL
;
75 unsigned cache_obj_size
= kmem_cache_size(cache
);
78 WARN_ON(cache_obj_size
< e_size
);
79 if (cache_obj_size
< e_size
)
82 /* e_count too big; would probably fail the allocation below anyways.
83 * for typical use cases, e_count should be few thousand at most. */
84 if (e_count
> LC_MAX_ACTIVE
)
87 slot
= kzalloc(e_count
* sizeof(struct hlist_head
*), GFP_KERNEL
);
90 element
= kzalloc(e_count
* sizeof(struct lc_element
*), GFP_KERNEL
);
94 lc
= kzalloc(sizeof(*lc
), GFP_KERNEL
);
98 INIT_LIST_HEAD(&lc
->in_use
);
99 INIT_LIST_HEAD(&lc
->lru
);
100 INIT_LIST_HEAD(&lc
->free
);
103 lc
->element_size
= e_size
;
104 lc
->element_off
= e_off
;
105 lc
->nr_elements
= e_count
;
106 lc
->new_number
= LC_FREE
;
107 lc
->lc_cache
= cache
;
108 lc
->lc_element
= element
;
111 /* preallocate all objects */
112 for (i
= 0; i
< e_count
; i
++) {
113 void *p
= kmem_cache_alloc(cache
, GFP_KERNEL
);
116 memset(p
, 0, lc
->element_size
);
119 e
->lc_number
= LC_FREE
;
120 list_add(&e
->list
, &lc
->free
);
126 /* else: could not allocate all elements, give up */
128 void *p
= element
[i
];
129 kmem_cache_free(cache
, p
- e_off
);
138 void lc_free_by_index(struct lru_cache
*lc
, unsigned i
)
140 void *p
= lc
->lc_element
[i
];
143 p
-= lc
->element_off
;
144 kmem_cache_free(lc
->lc_cache
, p
);
149 * lc_destroy - frees memory allocated by lc_create()
150 * @lc: the lru cache to destroy
152 void lc_destroy(struct lru_cache
*lc
)
157 for (i
= 0; i
< lc
->nr_elements
; i
++)
158 lc_free_by_index(lc
, i
);
159 kfree(lc
->lc_element
);
165 * lc_reset - does a full reset for @lc and the hash table slots.
166 * @lc: the lru cache to operate on
168 * It is roughly the equivalent of re-allocating a fresh lru_cache object,
169 * basically a short cut to lc_destroy(lc); lc = lc_create(...);
171 void lc_reset(struct lru_cache
*lc
)
175 INIT_LIST_HEAD(&lc
->in_use
);
176 INIT_LIST_HEAD(&lc
->lru
);
177 INIT_LIST_HEAD(&lc
->free
);
185 lc
->changing_element
= NULL
;
186 lc
->new_number
= LC_FREE
;
187 memset(lc
->lc_slot
, 0, sizeof(struct hlist_head
) * lc
->nr_elements
);
189 for (i
= 0; i
< lc
->nr_elements
; i
++) {
190 struct lc_element
*e
= lc
->lc_element
[i
];
192 p
-= lc
->element_off
;
193 memset(p
, 0, lc
->element_size
);
196 e
->lc_number
= LC_FREE
;
197 list_add(&e
->list
, &lc
->free
);
202 * lc_seq_printf_stats - print stats about @lc into @seq
203 * @seq: the seq_file to print into
204 * @lc: the lru cache to print statistics of
206 size_t lc_seq_printf_stats(struct seq_file
*seq
, struct lru_cache
*lc
)
209 * total calls to lc_get are
210 * (starving + hits + misses)
211 * misses include "dirty" count (update from an other thread in
212 * progress) and "changed", when this in fact lead to an successful
213 * update of the cache.
215 return seq_printf(seq
, "\t%s: used:%u/%u "
216 "hits:%lu misses:%lu starving:%lu dirty:%lu changed:%lu\n",
217 lc
->name
, lc
->used
, lc
->nr_elements
,
218 lc
->hits
, lc
->misses
, lc
->starving
, lc
->dirty
, lc
->changed
);
221 static struct hlist_head
*lc_hash_slot(struct lru_cache
*lc
, unsigned int enr
)
223 return lc
->lc_slot
+ (enr
% lc
->nr_elements
);
228 * lc_find - find element by label, if present in the hash table
229 * @lc: The lru_cache object
230 * @enr: element number
232 * Returns the pointer to an element, if the element with the requested
233 * "label" or element number is present in the hash table,
234 * or NULL if not found. Does not change the refcnt.
236 struct lc_element
*lc_find(struct lru_cache
*lc
, unsigned int enr
)
238 struct hlist_node
*n
;
239 struct lc_element
*e
;
242 BUG_ON(!lc
->nr_elements
);
243 hlist_for_each_entry(e
, n
, lc_hash_slot(lc
, enr
), colision
) {
244 if (e
->lc_number
== enr
)
250 /* returned element will be "recycled" immediately */
251 static struct lc_element
*lc_evict(struct lru_cache
*lc
)
254 struct lc_element
*e
;
256 if (list_empty(&lc
->lru
))
260 e
= list_entry(n
, struct lc_element
, list
);
262 PARANOIA_LC_ELEMENT(lc
, e
);
265 hlist_del(&e
->colision
);
270 * lc_del - removes an element from the cache
271 * @lc: The lru_cache object
272 * @e: The element to remove
274 * @e must be unused (refcnt == 0). Moves @e from "lru" to "free" list,
275 * sets @e->enr to %LC_FREE.
277 void lc_del(struct lru_cache
*lc
, struct lc_element
*e
)
280 PARANOIA_LC_ELEMENT(lc
, e
);
283 e
->lc_number
= LC_FREE
;
284 hlist_del_init(&e
->colision
);
285 list_move(&e
->list
, &lc
->free
);
289 static struct lc_element
*lc_get_unused_element(struct lru_cache
*lc
)
293 if (list_empty(&lc
->free
))
298 return list_entry(n
, struct lc_element
, list
);
301 static int lc_unused_element_available(struct lru_cache
*lc
)
303 if (!list_empty(&lc
->free
))
304 return 1; /* something on the free list */
305 if (!list_empty(&lc
->lru
))
306 return 1; /* something to evict */
313 * lc_get - get element by label, maybe change the active set
314 * @lc: the lru cache to operate on
315 * @enr: the label to look up
317 * Finds an element in the cache, increases its usage count,
318 * "touches" and returns it.
320 * In case the requested number is not present, it needs to be added to the
321 * cache. Therefore it is possible that an other element becomes evicted from
322 * the cache. In either case, the user is notified so he is able to e.g. keep
323 * a persistent log of the cache changes, and therefore the objects in use.
327 * The cache was marked %LC_STARVING,
328 * or the requested label was not in the active set
329 * and a changing transaction is still pending (@lc was marked %LC_DIRTY).
330 * Or no unused or free element could be recycled (@lc will be marked as
331 * %LC_STARVING, blocking further lc_get() operations).
333 * pointer to the element with the REQUESTED element number.
334 * In this case, it can be used right away
336 * pointer to an UNUSED element with some different element number,
337 * where that different number may also be %LC_FREE.
339 * In this case, the cache is marked %LC_DIRTY (blocking further changes),
340 * and the returned element pointer is removed from the lru list and
341 * hash collision chains. The user now should do whatever housekeeping
343 * Then he must call lc_changed(lc,element_pointer), to finish
346 * NOTE: The user needs to check the lc_number on EACH use, so he recognizes
347 * any cache set change.
349 struct lc_element
*lc_get(struct lru_cache
*lc
, unsigned int enr
)
351 struct lc_element
*e
;
354 if (lc
->flags
& LC_STARVING
) {
359 e
= lc_find(lc
, enr
);
362 if (e
->refcnt
++ == 0)
364 list_move(&e
->list
, &lc
->in_use
); /* Not evictable... */
370 /* In case there is nothing available and we can not kick out
371 * the LRU element, we have to wait ...
373 if (!lc_unused_element_available(lc
)) {
374 __set_bit(__LC_STARVING
, &lc
->flags
);
378 /* it was not present in the active set.
379 * we are going to recycle an unused (or even "free") element.
380 * user may need to commit a transaction to record that change.
381 * we serialize on flags & TF_DIRTY */
382 if (test_and_set_bit(__LC_DIRTY
, &lc
->flags
)) {
387 e
= lc_get_unused_element(lc
);
390 clear_bit(__LC_STARVING
, &lc
->flags
);
391 BUG_ON(++e
->refcnt
!= 1);
394 lc
->changing_element
= e
;
395 lc
->new_number
= enr
;
400 /* similar to lc_get,
401 * but only gets a new reference on an existing element.
402 * you either get the requested element, or NULL.
403 * will be consolidated into one function.
405 struct lc_element
*lc_try_get(struct lru_cache
*lc
, unsigned int enr
)
407 struct lc_element
*e
;
410 if (lc
->flags
& LC_STARVING
) {
415 e
= lc_find(lc
, enr
);
418 if (e
->refcnt
++ == 0)
420 list_move(&e
->list
, &lc
->in_use
); /* Not evictable... */
426 * lc_changed - tell @lc that the change has been recorded
427 * @lc: the lru cache to operate on
428 * @e: the element pending label change
430 void lc_changed(struct lru_cache
*lc
, struct lc_element
*e
)
433 BUG_ON(e
!= lc
->changing_element
);
434 PARANOIA_LC_ELEMENT(lc
, e
);
436 e
->lc_number
= lc
->new_number
;
437 list_add(&e
->list
, &lc
->in_use
);
438 hlist_add_head(&e
->colision
, lc_hash_slot(lc
, lc
->new_number
));
439 lc
->changing_element
= NULL
;
440 lc
->new_number
= LC_FREE
;
441 clear_bit(__LC_DIRTY
, &lc
->flags
);
442 smp_mb__after_clear_bit();
448 * lc_put - give up refcnt of @e
449 * @lc: the lru cache to operate on
450 * @e: the element to put
452 * If refcnt reaches zero, the element is moved to the lru list,
453 * and a %LC_STARVING (if set) is cleared.
454 * Returns the new (post-decrement) refcnt.
456 unsigned int lc_put(struct lru_cache
*lc
, struct lc_element
*e
)
459 PARANOIA_LC_ELEMENT(lc
, e
);
460 BUG_ON(e
->refcnt
== 0);
461 BUG_ON(e
== lc
->changing_element
);
462 if (--e
->refcnt
== 0) {
463 /* move it to the front of LRU. */
464 list_move(&e
->list
, &lc
->lru
);
466 clear_bit(__LC_STARVING
, &lc
->flags
);
467 smp_mb__after_clear_bit();
473 * lc_element_by_index
474 * @lc: the lru cache to operate on
475 * @i: the index of the element to return
477 struct lc_element
*lc_element_by_index(struct lru_cache
*lc
, unsigned i
)
479 BUG_ON(i
>= lc
->nr_elements
);
480 BUG_ON(lc
->lc_element
[i
] == NULL
);
481 BUG_ON(lc
->lc_element
[i
]->lc_index
!= i
);
482 return lc
->lc_element
[i
];
487 * @lc: the lru cache to operate on
488 * @e: the element to query for its index position in lc->element
490 unsigned int lc_index_of(struct lru_cache
*lc
, struct lc_element
*e
)
492 PARANOIA_LC_ELEMENT(lc
, e
);
497 * lc_set - associate index with label
498 * @lc: the lru cache to operate on
499 * @enr: the label to set
500 * @index: the element index to associate label with.
502 * Used to initialize the active set to some previously recorded state.
504 void lc_set(struct lru_cache
*lc
, unsigned int enr
, int index
)
506 struct lc_element
*e
;
508 if (index
< 0 || index
>= lc
->nr_elements
)
511 e
= lc_element_by_index(lc
, index
);
514 hlist_del_init(&e
->colision
);
515 hlist_add_head(&e
->colision
, lc_hash_slot(lc
, enr
));
516 list_move(&e
->list
, e
->refcnt
? &lc
->in_use
: &lc
->lru
);
520 * lc_dump - Dump a complete LRU cache to seq in textual form.
521 * @lc: the lru cache to operate on
522 * @seq: the &struct seq_file pointer to seq_printf into
523 * @utext: user supplied "heading" or other info
524 * @detail: function pointer the user may provide to dump further details
525 * of the object the lc_element is embedded in.
527 void lc_seq_dump_details(struct seq_file
*seq
, struct lru_cache
*lc
, char *utext
,
528 void (*detail
) (struct seq_file
*, struct lc_element
*))
530 unsigned int nr_elements
= lc
->nr_elements
;
531 struct lc_element
*e
;
534 seq_printf(seq
, "\tnn: lc_number refcnt %s\n ", utext
);
535 for (i
= 0; i
< nr_elements
; i
++) {
536 e
= lc_element_by_index(lc
, i
);
537 if (e
->lc_number
== LC_FREE
) {
538 seq_printf(seq
, "\t%2d: FREE\n", i
);
540 seq_printf(seq
, "\t%2d: %4u %4u ", i
,
541 e
->lc_number
, e
->refcnt
);
547 EXPORT_SYMBOL(lc_create
);
548 EXPORT_SYMBOL(lc_reset
);
549 EXPORT_SYMBOL(lc_destroy
);
550 EXPORT_SYMBOL(lc_set
);
551 EXPORT_SYMBOL(lc_del
);
552 EXPORT_SYMBOL(lc_try_get
);
553 EXPORT_SYMBOL(lc_find
);
554 EXPORT_SYMBOL(lc_get
);
555 EXPORT_SYMBOL(lc_put
);
556 EXPORT_SYMBOL(lc_changed
);
557 EXPORT_SYMBOL(lc_element_by_index
);
558 EXPORT_SYMBOL(lc_index_of
);
559 EXPORT_SYMBOL(lc_seq_printf_stats
);
560 EXPORT_SYMBOL(lc_seq_dump_details
);
This page took 0.043124 seconds and 5 git commands to generate.