Commit | Line | Data |
---|---|---|
86db1e29 JA |
1 | /* |
2 | * Functions related to io context handling | |
3 | */ | |
4 | #include <linux/kernel.h> | |
5 | #include <linux/module.h> | |
6 | #include <linux/init.h> | |
7 | #include <linux/bio.h> | |
8 | #include <linux/blkdev.h> | |
9 | #include <linux/bootmem.h> /* for max_pfn/max_low_pfn */ | |
5a0e3ad6 | 10 | #include <linux/slab.h> |
86db1e29 JA |
11 | |
12 | #include "blk.h" | |
13 | ||
14 | /* | |
15 | * For io context allocations | |
16 | */ | |
17 | static struct kmem_cache *iocontext_cachep; | |
18 | ||
19 | static void cfq_dtor(struct io_context *ioc) | |
20 | { | |
ffc4e759 JA |
21 | if (!hlist_empty(&ioc->cic_list)) { |
22 | struct cfq_io_context *cic; | |
23 | ||
e2bd9678 | 24 | cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context, |
ffc4e759 JA |
25 | cic_list); |
26 | cic->dtor(ioc); | |
27 | } | |
86db1e29 JA |
28 | } |
29 | ||
30 | /* | |
31 | * IO Context helper functions. put_io_context() returns 1 if there are no | |
32 | * more users of this io context, 0 otherwise. | |
33 | */ | |
34 | int put_io_context(struct io_context *ioc) | |
35 | { | |
36 | if (ioc == NULL) | |
37 | return 1; | |
38 | ||
d9c7d394 | 39 | BUG_ON(atomic_long_read(&ioc->refcount) == 0); |
86db1e29 | 40 | |
d9c7d394 | 41 | if (atomic_long_dec_and_test(&ioc->refcount)) { |
86db1e29 | 42 | rcu_read_lock(); |
86db1e29 | 43 | cfq_dtor(ioc); |
07416d29 | 44 | rcu_read_unlock(); |
86db1e29 JA |
45 | |
46 | kmem_cache_free(iocontext_cachep, ioc); | |
47 | return 1; | |
48 | } | |
49 | return 0; | |
50 | } | |
51 | EXPORT_SYMBOL(put_io_context); | |
52 | ||
53 | static void cfq_exit(struct io_context *ioc) | |
54 | { | |
86db1e29 | 55 | rcu_read_lock(); |
86db1e29 | 56 | |
ffc4e759 JA |
57 | if (!hlist_empty(&ioc->cic_list)) { |
58 | struct cfq_io_context *cic; | |
59 | ||
e2bd9678 | 60 | cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context, |
ffc4e759 JA |
61 | cic_list); |
62 | cic->exit(ioc); | |
63 | } | |
64 | rcu_read_unlock(); | |
86db1e29 JA |
65 | } |
66 | ||
27667c99 | 67 | /* Called by the exiting task */ |
b69f2292 | 68 | void exit_io_context(struct task_struct *task) |
86db1e29 JA |
69 | { |
70 | struct io_context *ioc; | |
71 | ||
b69f2292 LR |
72 | task_lock(task); |
73 | ioc = task->io_context; | |
74 | task->io_context = NULL; | |
75 | task_unlock(task); | |
86db1e29 | 76 | |
27667c99 | 77 | if (atomic_dec_and_test(&ioc->nr_tasks)) |
86db1e29 JA |
78 | cfq_exit(ioc); |
79 | ||
61cc74fb | 80 | put_io_context(ioc); |
86db1e29 JA |
81 | } |
82 | ||
83 | struct io_context *alloc_io_context(gfp_t gfp_flags, int node) | |
84 | { | |
df415656 | 85 | struct io_context *ioc; |
86db1e29 | 86 | |
df415656 PB |
87 | ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node); |
88 | if (ioc) { | |
89 | atomic_long_set(&ioc->refcount, 1); | |
90 | atomic_set(&ioc->nr_tasks, 1); | |
91 | spin_lock_init(&ioc->lock); | |
92 | ioc->ioprio_changed = 0; | |
93 | ioc->ioprio = 0; | |
94 | ioc->last_waited = 0; /* doesn't matter... */ | |
95 | ioc->nr_batch_requests = 0; /* because this is 0 */ | |
96 | INIT_RADIX_TREE(&ioc->radix_root, GFP_ATOMIC | __GFP_HIGH); | |
97 | INIT_HLIST_HEAD(&ioc->cic_list); | |
98 | ioc->ioc_data = NULL; | |
4cbadbd1 | 99 | #if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_BLK_CGROUP_MODULE) |
df415656 | 100 | ioc->cgroup_changed = 0; |
4cbadbd1 | 101 | #endif |
86db1e29 JA |
102 | } |
103 | ||
df415656 | 104 | return ioc; |
86db1e29 JA |
105 | } |
106 | ||
107 | /* | |
108 | * If the current task has no IO context then create one and initialise it. | |
109 | * Otherwise, return its existing IO context. | |
110 | * | |
111 | * This returned IO context doesn't have a specifically elevated refcount, | |
112 | * but since the current task itself holds a reference, the context can be | |
113 | * used in general code, so long as it stays within `current` context. | |
114 | */ | |
115 | struct io_context *current_io_context(gfp_t gfp_flags, int node) | |
116 | { | |
117 | struct task_struct *tsk = current; | |
118 | struct io_context *ret; | |
119 | ||
120 | ret = tsk->io_context; | |
121 | if (likely(ret)) | |
122 | return ret; | |
123 | ||
124 | ret = alloc_io_context(gfp_flags, node); | |
125 | if (ret) { | |
126 | /* make sure set_task_ioprio() sees the settings above */ | |
127 | smp_wmb(); | |
128 | tsk->io_context = ret; | |
129 | } | |
130 | ||
131 | return ret; | |
132 | } | |
133 | ||
134 | /* | |
135 | * If the current task has no IO context then create one and initialise it. | |
136 | * If it does have a context, take a ref on it. | |
137 | * | |
138 | * This is always called in the context of the task which submitted the I/O. | |
139 | */ | |
140 | struct io_context *get_io_context(gfp_t gfp_flags, int node) | |
141 | { | |
df415656 | 142 | struct io_context *ioc = NULL; |
86db1e29 JA |
143 | |
144 | /* | |
145 | * Check for unlikely race with exiting task. ioc ref count is | |
146 | * zero when ioc is being detached. | |
147 | */ | |
148 | do { | |
df415656 PB |
149 | ioc = current_io_context(gfp_flags, node); |
150 | if (unlikely(!ioc)) | |
86db1e29 | 151 | break; |
df415656 | 152 | } while (!atomic_long_inc_not_zero(&ioc->refcount)); |
86db1e29 | 153 | |
df415656 | 154 | return ioc; |
86db1e29 JA |
155 | } |
156 | EXPORT_SYMBOL(get_io_context); | |
157 | ||
13341598 | 158 | static int __init blk_ioc_init(void) |
86db1e29 JA |
159 | { |
160 | iocontext_cachep = kmem_cache_create("blkdev_ioc", | |
161 | sizeof(struct io_context), 0, SLAB_PANIC, NULL); | |
162 | return 0; | |
163 | } | |
164 | subsys_initcall(blk_ioc_init); |