Commit | Line | Data |
---|---|---|
86db1e29 JA |
1 | /* |
2 | * Functions related to io context handling | |
3 | */ | |
4 | #include <linux/kernel.h> | |
5 | #include <linux/module.h> | |
6 | #include <linux/init.h> | |
7 | #include <linux/bio.h> | |
8 | #include <linux/blkdev.h> | |
9 | #include <linux/bootmem.h> /* for max_pfn/max_low_pfn */ | |
10 | ||
11 | #include "blk.h" | |
12 | ||
13 | /* | |
14 | * For io context allocations | |
15 | */ | |
16 | static struct kmem_cache *iocontext_cachep; | |
17 | ||
18 | static void cfq_dtor(struct io_context *ioc) | |
19 | { | |
20 | struct cfq_io_context *cic[1]; | |
21 | int r; | |
22 | ||
23 | /* | |
24 | * We don't have a specific key to lookup with, so use the gang | |
25 | * lookup to just retrieve the first item stored. The cfq exit | |
26 | * function will iterate the full tree, so any member will do. | |
27 | */ | |
28 | r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1); | |
29 | if (r > 0) | |
30 | cic[0]->dtor(ioc); | |
31 | } | |
32 | ||
33 | /* | |
34 | * IO Context helper functions. put_io_context() returns 1 if there are no | |
35 | * more users of this io context, 0 otherwise. | |
36 | */ | |
37 | int put_io_context(struct io_context *ioc) | |
38 | { | |
39 | if (ioc == NULL) | |
40 | return 1; | |
41 | ||
42 | BUG_ON(atomic_read(&ioc->refcount) == 0); | |
43 | ||
44 | if (atomic_dec_and_test(&ioc->refcount)) { | |
45 | rcu_read_lock(); | |
46 | if (ioc->aic && ioc->aic->dtor) | |
47 | ioc->aic->dtor(ioc->aic); | |
48 | rcu_read_unlock(); | |
49 | cfq_dtor(ioc); | |
50 | ||
51 | kmem_cache_free(iocontext_cachep, ioc); | |
52 | return 1; | |
53 | } | |
54 | return 0; | |
55 | } | |
56 | EXPORT_SYMBOL(put_io_context); | |
57 | ||
58 | static void cfq_exit(struct io_context *ioc) | |
59 | { | |
60 | struct cfq_io_context *cic[1]; | |
61 | int r; | |
62 | ||
63 | rcu_read_lock(); | |
64 | /* | |
65 | * See comment for cfq_dtor() | |
66 | */ | |
67 | r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1); | |
68 | rcu_read_unlock(); | |
69 | ||
70 | if (r > 0) | |
71 | cic[0]->exit(ioc); | |
72 | } | |
73 | ||
74 | /* Called by the exitting task */ | |
75 | void exit_io_context(void) | |
76 | { | |
77 | struct io_context *ioc; | |
78 | ||
79 | task_lock(current); | |
80 | ioc = current->io_context; | |
81 | current->io_context = NULL; | |
82 | task_unlock(current); | |
83 | ||
84 | if (atomic_dec_and_test(&ioc->nr_tasks)) { | |
85 | if (ioc->aic && ioc->aic->exit) | |
86 | ioc->aic->exit(ioc->aic); | |
87 | cfq_exit(ioc); | |
88 | ||
89 | put_io_context(ioc); | |
90 | } | |
91 | } | |
92 | ||
93 | struct io_context *alloc_io_context(gfp_t gfp_flags, int node) | |
94 | { | |
95 | struct io_context *ret; | |
96 | ||
97 | ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node); | |
98 | if (ret) { | |
99 | atomic_set(&ret->refcount, 1); | |
100 | atomic_set(&ret->nr_tasks, 1); | |
101 | spin_lock_init(&ret->lock); | |
102 | ret->ioprio_changed = 0; | |
103 | ret->ioprio = 0; | |
104 | ret->last_waited = jiffies; /* doesn't matter... */ | |
105 | ret->nr_batch_requests = 0; /* because this is 0 */ | |
106 | ret->aic = NULL; | |
107 | INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH); | |
108 | ret->ioc_data = NULL; | |
109 | } | |
110 | ||
111 | return ret; | |
112 | } | |
113 | ||
114 | /* | |
115 | * If the current task has no IO context then create one and initialise it. | |
116 | * Otherwise, return its existing IO context. | |
117 | * | |
118 | * This returned IO context doesn't have a specifically elevated refcount, | |
119 | * but since the current task itself holds a reference, the context can be | |
120 | * used in general code, so long as it stays within `current` context. | |
121 | */ | |
122 | struct io_context *current_io_context(gfp_t gfp_flags, int node) | |
123 | { | |
124 | struct task_struct *tsk = current; | |
125 | struct io_context *ret; | |
126 | ||
127 | ret = tsk->io_context; | |
128 | if (likely(ret)) | |
129 | return ret; | |
130 | ||
131 | ret = alloc_io_context(gfp_flags, node); | |
132 | if (ret) { | |
133 | /* make sure set_task_ioprio() sees the settings above */ | |
134 | smp_wmb(); | |
135 | tsk->io_context = ret; | |
136 | } | |
137 | ||
138 | return ret; | |
139 | } | |
140 | ||
141 | /* | |
142 | * If the current task has no IO context then create one and initialise it. | |
143 | * If it does have a context, take a ref on it. | |
144 | * | |
145 | * This is always called in the context of the task which submitted the I/O. | |
146 | */ | |
147 | struct io_context *get_io_context(gfp_t gfp_flags, int node) | |
148 | { | |
149 | struct io_context *ret = NULL; | |
150 | ||
151 | /* | |
152 | * Check for unlikely race with exiting task. ioc ref count is | |
153 | * zero when ioc is being detached. | |
154 | */ | |
155 | do { | |
156 | ret = current_io_context(gfp_flags, node); | |
157 | if (unlikely(!ret)) | |
158 | break; | |
159 | } while (!atomic_inc_not_zero(&ret->refcount)); | |
160 | ||
161 | return ret; | |
162 | } | |
163 | EXPORT_SYMBOL(get_io_context); | |
164 | ||
165 | void copy_io_context(struct io_context **pdst, struct io_context **psrc) | |
166 | { | |
167 | struct io_context *src = *psrc; | |
168 | struct io_context *dst = *pdst; | |
169 | ||
170 | if (src) { | |
171 | BUG_ON(atomic_read(&src->refcount) == 0); | |
172 | atomic_inc(&src->refcount); | |
173 | put_io_context(dst); | |
174 | *pdst = src; | |
175 | } | |
176 | } | |
177 | EXPORT_SYMBOL(copy_io_context); | |
178 | ||
179 | void swap_io_context(struct io_context **ioc1, struct io_context **ioc2) | |
180 | { | |
181 | struct io_context *temp; | |
182 | temp = *ioc1; | |
183 | *ioc1 = *ioc2; | |
184 | *ioc2 = temp; | |
185 | } | |
186 | EXPORT_SYMBOL(swap_io_context); | |
187 | ||
188 | int __init blk_ioc_init(void) | |
189 | { | |
190 | iocontext_cachep = kmem_cache_create("blkdev_ioc", | |
191 | sizeof(struct io_context), 0, SLAB_PANIC, NULL); | |
192 | return 0; | |
193 | } | |
194 | subsys_initcall(blk_ioc_init); |