1 #ifndef _LINUX_SLUB_DEF_H
2 #define _LINUX_SLUB_DEF_H
5 * SLUB : A Slab allocator without object queues.
7 * (C) 2007 SGI, Christoph Lameter
9 #include <linux/types.h>
10 #include <linux/gfp.h>
11 #include <linux/bug.h>
12 #include <linux/workqueue.h>
13 #include <linux/kobject.h>
15 #include <linux/kmemleak.h>
18 ALLOC_FASTPATH
, /* Allocation from cpu slab */
19 ALLOC_SLOWPATH
, /* Allocation by getting a new cpu slab */
20 FREE_FASTPATH
, /* Free to cpu slub */
21 FREE_SLOWPATH
, /* Freeing not to cpu slab */
22 FREE_FROZEN
, /* Freeing to frozen slab */
23 FREE_ADD_PARTIAL
, /* Freeing moves slab to partial list */
24 FREE_REMOVE_PARTIAL
, /* Freeing removes last object */
25 ALLOC_FROM_PARTIAL
, /* Cpu slab acquired from node partial list */
26 ALLOC_SLAB
, /* Cpu slab acquired from page allocator */
27 ALLOC_REFILL
, /* Refill cpu slab from slab freelist */
28 ALLOC_NODE_MISMATCH
, /* Switching cpu slab */
29 FREE_SLAB
, /* Slab freed to the page allocator */
30 CPUSLAB_FLUSH
, /* Abandoning of the cpu slab */
31 DEACTIVATE_FULL
, /* Cpu slab was full when deactivated */
32 DEACTIVATE_EMPTY
, /* Cpu slab was empty when deactivated */
33 DEACTIVATE_TO_HEAD
, /* Cpu slab was moved to the head of partials */
34 DEACTIVATE_TO_TAIL
, /* Cpu slab was moved to the tail of partials */
35 DEACTIVATE_REMOTE_FREES
,/* Slab contained remotely freed objects */
36 DEACTIVATE_BYPASS
, /* Implicit deactivation */
37 ORDER_FALLBACK
, /* Number of times fallback was necessary */
38 CMPXCHG_DOUBLE_CPU_FAIL
,/* Failure of this_cpu_cmpxchg_double */
39 CMPXCHG_DOUBLE_FAIL
, /* Number of times that cmpxchg double did not match */
40 CPU_PARTIAL_ALLOC
, /* Used cpu partial on alloc */
41 CPU_PARTIAL_FREE
, /* Refill cpu partial on free */
42 CPU_PARTIAL_NODE
, /* Refill cpu partial from node partial */
43 CPU_PARTIAL_DRAIN
, /* Drain cpu partial to node partial */
46 struct kmem_cache_cpu
{
47 void **freelist
; /* Pointer to next available object */
48 unsigned long tid
; /* Globally unique transaction id */
49 struct page
*page
; /* The slab from which we are allocating */
50 struct page
*partial
; /* Partially allocated frozen slabs */
51 int node
; /* The node of the page (or -1 for debug) */
52 #ifdef CONFIG_SLUB_STATS
53 unsigned stat
[NR_SLUB_STAT_ITEMS
];
57 struct kmem_cache_node
{
58 spinlock_t list_lock
; /* Protect partial list and nr_partial */
59 unsigned long nr_partial
;
60 struct list_head partial
;
61 #ifdef CONFIG_SLUB_DEBUG
62 atomic_long_t nr_slabs
;
63 atomic_long_t total_objects
;
64 struct list_head full
;
69 * Word size structure that can be atomically updated or read and that
70 * contains both the order and the number of objects that a slab of the
71 * given order would contain.
73 struct kmem_cache_order_objects
{
78 * Slab cache management.
81 struct kmem_cache_cpu __percpu
*cpu_slab
;
82 /* Used for retriving partial slabs etc */
84 unsigned long min_partial
;
85 int size
; /* The size of an object including meta data */
86 int objsize
; /* The size of an object without meta data */
87 int offset
; /* Free pointer offset. */
88 int cpu_partial
; /* Number of per cpu partial objects to keep around */
89 struct kmem_cache_order_objects oo
;
91 /* Allocation and freeing of slabs */
92 struct kmem_cache_order_objects max
;
93 struct kmem_cache_order_objects min
;
94 gfp_t allocflags
; /* gfp flags to use on each alloc */
95 int refcount
; /* Refcount for slab cache destroy */
97 int inuse
; /* Offset to metadata */
98 int align
; /* Alignment */
99 int reserved
; /* Reserved bytes at the end of slabs */
100 const char *name
; /* Name (only for display!) */
101 struct list_head list
; /* List of slab caches */
103 struct kobject kobj
; /* For sysfs */
108 * Defragmentation by allocating from a remote node.
110 int remote_node_defrag_ratio
;
112 struct kmem_cache_node
*node
[MAX_NUMNODES
];
118 #if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
119 #define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
121 #define KMALLOC_MIN_SIZE 8
124 #define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
127 * Maximum kmalloc object size handled by SLUB. Larger object allocations
128 * are passed through to the page allocator. The page allocator "fastpath"
129 * is relatively slow so we need this value sufficiently high so that
130 * performance critical objects are allocated through the SLUB fastpath.
132 * This should be dropped to PAGE_SIZE / 2 once the page allocator
133 * "fastpath" becomes competitive with the slab allocator fastpaths.
135 #define SLUB_MAX_SIZE (2 * PAGE_SIZE)
137 #define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2)
139 #ifdef CONFIG_ZONE_DMA
140 #define SLUB_DMA __GFP_DMA
142 /* Disable DMA functionality */
143 #define SLUB_DMA (__force gfp_t)0
147 * We keep the general caches in an array of slab caches that are used for
148 * 2^x bytes of allocations.
150 extern struct kmem_cache
*kmalloc_caches
[SLUB_PAGE_SHIFT
];
153 * Sorry that the following has to be that ugly but some versions of GCC
154 * have trouble with constant propagation and loops.
156 static __always_inline
int kmalloc_index(size_t size
)
161 if (size
<= KMALLOC_MIN_SIZE
)
162 return KMALLOC_SHIFT_LOW
;
164 if (KMALLOC_MIN_SIZE
<= 32 && size
> 64 && size
<= 96)
166 if (KMALLOC_MIN_SIZE
<= 64 && size
> 128 && size
<= 192)
168 if (size
<= 8) return 3;
169 if (size
<= 16) return 4;
170 if (size
<= 32) return 5;
171 if (size
<= 64) return 6;
172 if (size
<= 128) return 7;
173 if (size
<= 256) return 8;
174 if (size
<= 512) return 9;
175 if (size
<= 1024) return 10;
176 if (size
<= 2 * 1024) return 11;
177 if (size
<= 4 * 1024) return 12;
179 * The following is only needed to support architectures with a larger page
180 * size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page
181 * size we would have to go up to 128k.
183 if (size
<= 8 * 1024) return 13;
184 if (size
<= 16 * 1024) return 14;
185 if (size
<= 32 * 1024) return 15;
186 if (size
<= 64 * 1024) return 16;
187 if (size
<= 128 * 1024) return 17;
188 if (size
<= 256 * 1024) return 18;
189 if (size
<= 512 * 1024) return 19;
190 if (size
<= 1024 * 1024) return 20;
191 if (size
<= 2 * 1024 * 1024) return 21;
193 return -1; /* Will never be reached */
196 * What we really wanted to do and cannot do because of compiler issues is:
198 * for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
199 * if (size <= (1 << i))
205 * Find the slab cache for a given combination of allocation flags and size.
207 * This ought to end up with a global pointer to the right cache
210 static __always_inline
struct kmem_cache
*kmalloc_slab(size_t size
)
212 int index
= kmalloc_index(size
);
217 return kmalloc_caches
[index
];
220 void *kmem_cache_alloc(struct kmem_cache
*, gfp_t
);
221 void *__kmalloc(size_t size
, gfp_t flags
);
223 static __always_inline
void *
224 kmalloc_order(size_t size
, gfp_t flags
, unsigned int order
)
226 void *ret
= (void *) __get_free_pages(flags
| __GFP_COMP
, order
);
227 kmemleak_alloc(ret
, size
, 1, flags
);
232 * Calling this on allocated memory will check that the memory
233 * is expected to be in use, and print warnings if not.
235 #ifdef CONFIG_SLUB_DEBUG
236 extern bool verify_mem_not_deleted(const void *x
);
238 static inline bool verify_mem_not_deleted(const void *x
)
244 #ifdef CONFIG_TRACING
246 kmem_cache_alloc_trace(struct kmem_cache
*s
, gfp_t gfpflags
, size_t size
);
247 extern void *kmalloc_order_trace(size_t size
, gfp_t flags
, unsigned int order
);
249 static __always_inline
void *
250 kmem_cache_alloc_trace(struct kmem_cache
*s
, gfp_t gfpflags
, size_t size
)
252 return kmem_cache_alloc(s
, gfpflags
);
255 static __always_inline
void *
256 kmalloc_order_trace(size_t size
, gfp_t flags
, unsigned int order
)
258 return kmalloc_order(size
, flags
, order
);
262 static __always_inline
void *kmalloc_large(size_t size
, gfp_t flags
)
264 unsigned int order
= get_order(size
);
265 return kmalloc_order_trace(size
, flags
, order
);
268 static __always_inline
void *kmalloc(size_t size
, gfp_t flags
)
270 if (__builtin_constant_p(size
)) {
271 if (size
> SLUB_MAX_SIZE
)
272 return kmalloc_large(size
, flags
);
274 if (!(flags
& SLUB_DMA
)) {
275 struct kmem_cache
*s
= kmalloc_slab(size
);
278 return ZERO_SIZE_PTR
;
280 return kmem_cache_alloc_trace(s
, flags
, size
);
283 return __kmalloc(size
, flags
);
287 void *__kmalloc_node(size_t size
, gfp_t flags
, int node
);
288 void *kmem_cache_alloc_node(struct kmem_cache
*, gfp_t flags
, int node
);
290 #ifdef CONFIG_TRACING
291 extern void *kmem_cache_alloc_node_trace(struct kmem_cache
*s
,
293 int node
, size_t size
);
295 static __always_inline
void *
296 kmem_cache_alloc_node_trace(struct kmem_cache
*s
,
298 int node
, size_t size
)
300 return kmem_cache_alloc_node(s
, gfpflags
, node
);
304 static __always_inline
void *kmalloc_node(size_t size
, gfp_t flags
, int node
)
306 if (__builtin_constant_p(size
) &&
307 size
<= SLUB_MAX_SIZE
&& !(flags
& SLUB_DMA
)) {
308 struct kmem_cache
*s
= kmalloc_slab(size
);
311 return ZERO_SIZE_PTR
;
313 return kmem_cache_alloc_node_trace(s
, flags
, node
, size
);
315 return __kmalloc_node(size
, flags
, node
);
319 #endif /* _LINUX_SLUB_DEF_H */