Commit | Line | Data |
---|---|---|
d7e09d03 PT |
1 | /* |
2 | * GPL HEADER START | |
3 | * | |
4 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 only, | |
8 | * as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License version 2 for more details (a copy is included | |
14 | * in the LICENSE file that accompanied this code). | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * version 2 along with this program; If not, see | |
6a5b99a4 | 18 | * http://www.gnu.org/licenses/gpl-2.0.html |
d7e09d03 | 19 | * |
d7e09d03 PT |
20 | * GPL HEADER END |
21 | */ | |
22 | /* | |
23 | * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. | |
24 | * Use is subject to license terms. | |
25 | * | |
1dc563a6 | 26 | * Copyright (c) 2011, 2015, Intel Corporation. |
d7e09d03 PT |
27 | */ |
28 | /* | |
29 | * This file is part of Lustre, http://www.lustre.org/ | |
30 | * Lustre is a trademark of Sun Microsystems, Inc. | |
31 | * | |
32 | * Implementation of cl_io for LOV layer. | |
33 | * | |
34 | * Author: Nikita Danilov <nikita.danilov@sun.com> | |
35 | * Author: Jinshan Xiong <jinshan.xiong@whamcloud.com> | |
36 | */ | |
37 | ||
38 | #define DEBUG_SUBSYSTEM S_LOV | |
39 | ||
40 | #include "lov_cl_internal.h" | |
41 | ||
42 | /** \addtogroup lov | |
43 | * @{ | |
44 | */ | |
45 | ||
46 | static inline void lov_sub_enter(struct lov_io_sub *sub) | |
47 | { | |
48 | sub->sub_reenter++; | |
49 | } | |
c9f6bb96 | 50 | |
d7e09d03 PT |
51 | static inline void lov_sub_exit(struct lov_io_sub *sub) |
52 | { | |
53 | sub->sub_reenter--; | |
54 | } | |
55 | ||
56 | static void lov_io_sub_fini(const struct lu_env *env, struct lov_io *lio, | |
57 | struct lov_io_sub *sub) | |
58 | { | |
00697c43 | 59 | if (sub->sub_io) { |
d7e09d03 PT |
60 | if (sub->sub_io_initialized) { |
61 | lov_sub_enter(sub); | |
62 | cl_io_fini(sub->sub_env, sub->sub_io); | |
63 | lov_sub_exit(sub); | |
64 | sub->sub_io_initialized = 0; | |
65 | lio->lis_active_subios--; | |
66 | } | |
67 | if (sub->sub_stripe == lio->lis_single_subio_index) | |
68 | lio->lis_single_subio_index = -1; | |
69 | else if (!sub->sub_borrowed) | |
840c94d5 | 70 | kfree(sub->sub_io); |
d7e09d03 PT |
71 | sub->sub_io = NULL; |
72 | } | |
00697c43 | 73 | if (!IS_ERR_OR_NULL(sub->sub_env)) { |
d7e09d03 PT |
74 | if (!sub->sub_borrowed) |
75 | cl_env_put(sub->sub_env, &sub->sub_refcheck); | |
76 | sub->sub_env = NULL; | |
77 | } | |
d7e09d03 PT |
78 | } |
79 | ||
80 | static void lov_io_sub_inherit(struct cl_io *io, struct lov_io *lio, | |
81 | int stripe, loff_t start, loff_t end) | |
82 | { | |
83 | struct lov_stripe_md *lsm = lio->lis_object->lo_lsm; | |
84 | struct cl_io *parent = lio->lis_cl.cis_io; | |
85 | ||
5f3a68f9 | 86 | switch (io->ci_type) { |
d7e09d03 PT |
87 | case CIT_SETATTR: { |
88 | io->u.ci_setattr.sa_attr = parent->u.ci_setattr.sa_attr; | |
89 | io->u.ci_setattr.sa_valid = parent->u.ci_setattr.sa_valid; | |
d7e09d03 PT |
90 | if (cl_io_is_trunc(io)) { |
91 | loff_t new_size = parent->u.ci_setattr.sa_attr.lvb_size; | |
92 | ||
93 | new_size = lov_size_to_stripe(lsm, new_size, stripe); | |
94 | io->u.ci_setattr.sa_attr.lvb_size = new_size; | |
95 | } | |
96 | break; | |
97 | } | |
98 | case CIT_FAULT: { | |
99 | struct cl_object *obj = parent->ci_obj; | |
100 | loff_t off = cl_offset(obj, parent->u.ci_fault.ft_index); | |
101 | ||
102 | io->u.ci_fault = parent->u.ci_fault; | |
103 | off = lov_size_to_stripe(lsm, off, stripe); | |
104 | io->u.ci_fault.ft_index = cl_index(obj, off); | |
105 | break; | |
106 | } | |
107 | case CIT_FSYNC: { | |
108 | io->u.ci_fsync.fi_start = start; | |
109 | io->u.ci_fsync.fi_end = end; | |
d7e09d03 PT |
110 | io->u.ci_fsync.fi_fid = parent->u.ci_fsync.fi_fid; |
111 | io->u.ci_fsync.fi_mode = parent->u.ci_fsync.fi_mode; | |
112 | break; | |
113 | } | |
114 | case CIT_READ: | |
115 | case CIT_WRITE: { | |
116 | io->u.ci_wr.wr_sync = cl_io_is_sync_write(parent); | |
117 | if (cl_io_is_append(parent)) { | |
118 | io->u.ci_wr.wr_append = 1; | |
119 | } else { | |
120 | io->u.ci_rw.crw_pos = start; | |
121 | io->u.ci_rw.crw_count = end - start; | |
122 | } | |
123 | break; | |
124 | } | |
125 | default: | |
126 | break; | |
127 | } | |
128 | } | |
129 | ||
130 | static int lov_io_sub_init(const struct lu_env *env, struct lov_io *lio, | |
131 | struct lov_io_sub *sub) | |
132 | { | |
133 | struct lov_object *lov = lio->lis_object; | |
134 | struct lov_device *ld = lu2lov_dev(lov2cl(lov)->co_lu.lo_dev); | |
135 | struct cl_io *sub_io; | |
136 | struct cl_object *sub_obj; | |
137 | struct cl_io *io = lio->lis_cl.cis_io; | |
138 | ||
139 | int stripe = sub->sub_stripe; | |
140 | int result; | |
141 | ||
00697c43 OD |
142 | LASSERT(!sub->sub_io); |
143 | LASSERT(!sub->sub_env); | |
d7e09d03 | 144 | LASSERT(sub->sub_stripe < lio->lis_stripe_count); |
d7e09d03 | 145 | |
00697c43 | 146 | if (unlikely(!lov_r0(lov)->lo_sub[stripe])) |
397632e4 YS |
147 | return -EIO; |
148 | ||
d7e09d03 PT |
149 | result = 0; |
150 | sub->sub_io_initialized = 0; | |
151 | sub->sub_borrowed = 0; | |
152 | ||
153 | if (lio->lis_mem_frozen) { | |
154 | LASSERT(mutex_is_locked(&ld->ld_mutex)); | |
155 | sub->sub_io = &ld->ld_emrg[stripe]->emrg_subio; | |
156 | sub->sub_env = ld->ld_emrg[stripe]->emrg_env; | |
157 | sub->sub_borrowed = 1; | |
158 | } else { | |
159 | void *cookie; | |
160 | ||
161 | /* obtain new environment */ | |
162 | cookie = cl_env_reenter(); | |
163 | sub->sub_env = cl_env_get(&sub->sub_refcheck); | |
164 | cl_env_reexit(cookie); | |
165 | if (IS_ERR(sub->sub_env)) | |
166 | result = PTR_ERR(sub->sub_env); | |
167 | ||
168 | if (result == 0) { | |
169 | /* | |
170 | * First sub-io. Use ->lis_single_subio to | |
171 | * avoid dynamic allocation. | |
172 | */ | |
173 | if (lio->lis_active_subios == 0) { | |
174 | sub->sub_io = &lio->lis_single_subio; | |
175 | lio->lis_single_subio_index = stripe; | |
176 | } else { | |
840c94d5 JL |
177 | sub->sub_io = kzalloc(sizeof(*sub->sub_io), |
178 | GFP_NOFS); | |
36a86fe6 | 179 | if (!sub->sub_io) |
d7e09d03 PT |
180 | result = -ENOMEM; |
181 | } | |
182 | } | |
183 | } | |
184 | ||
185 | if (result == 0) { | |
186 | sub_obj = lovsub2cl(lov_r0(lov)->lo_sub[stripe]); | |
187 | sub_io = sub->sub_io; | |
188 | ||
189 | sub_io->ci_obj = sub_obj; | |
190 | sub_io->ci_result = 0; | |
191 | ||
192 | sub_io->ci_parent = io; | |
193 | sub_io->ci_lockreq = io->ci_lockreq; | |
194 | sub_io->ci_type = io->ci_type; | |
195 | sub_io->ci_no_srvlock = io->ci_no_srvlock; | |
ec9bca9c | 196 | sub_io->ci_noatime = io->ci_noatime; |
d7e09d03 PT |
197 | |
198 | lov_sub_enter(sub); | |
199 | result = cl_io_sub_init(sub->sub_env, sub_io, | |
200 | io->ci_type, sub_obj); | |
201 | lov_sub_exit(sub); | |
202 | if (result >= 0) { | |
203 | lio->lis_active_subios++; | |
204 | sub->sub_io_initialized = 1; | |
205 | result = 0; | |
206 | } | |
207 | } | |
208 | if (result != 0) | |
209 | lov_io_sub_fini(env, lio, sub); | |
0a3bdb00 | 210 | return result; |
d7e09d03 PT |
211 | } |
212 | ||
213 | struct lov_io_sub *lov_sub_get(const struct lu_env *env, | |
214 | struct lov_io *lio, int stripe) | |
215 | { | |
216 | int rc; | |
217 | struct lov_io_sub *sub = &lio->lis_subs[stripe]; | |
218 | ||
219 | LASSERT(stripe < lio->lis_stripe_count); | |
d7e09d03 PT |
220 | |
221 | if (!sub->sub_io_initialized) { | |
222 | sub->sub_stripe = stripe; | |
223 | rc = lov_io_sub_init(env, lio, sub); | |
da5ecb4d | 224 | } else { |
d7e09d03 | 225 | rc = 0; |
da5ecb4d | 226 | } |
d7e09d03 PT |
227 | if (rc == 0) |
228 | lov_sub_enter(sub); | |
229 | else | |
230 | sub = ERR_PTR(rc); | |
0a3bdb00 | 231 | return sub; |
d7e09d03 PT |
232 | } |
233 | ||
234 | void lov_sub_put(struct lov_io_sub *sub) | |
235 | { | |
236 | lov_sub_exit(sub); | |
237 | } | |
238 | ||
239 | /***************************************************************************** | |
240 | * | |
241 | * Lov io operations. | |
242 | * | |
243 | */ | |
244 | ||
fd7444fe | 245 | int lov_page_stripe(const struct cl_page *page) |
d7e09d03 | 246 | { |
7addf402 | 247 | const struct cl_page_slice *slice; |
d7e09d03 | 248 | |
96c53363 | 249 | slice = cl_page_at(page, &lov_device_type); |
7addf402 JX |
250 | LASSERT(slice->cpl_obj); |
251 | ||
96c53363 | 252 | return cl2lov_page(slice)->lps_stripe; |
d7e09d03 PT |
253 | } |
254 | ||
255 | struct lov_io_sub *lov_page_subio(const struct lu_env *env, struct lov_io *lio, | |
256 | const struct cl_page_slice *slice) | |
257 | { | |
258 | struct lov_stripe_md *lsm = lio->lis_object->lo_lsm; | |
259 | struct cl_page *page = slice->cpl_page; | |
260 | int stripe; | |
261 | ||
00697c43 | 262 | LASSERT(lio->lis_cl.cis_io); |
d7e09d03 | 263 | LASSERT(cl2lov(slice->cpl_obj) == lio->lis_object); |
00697c43 | 264 | LASSERT(lsm); |
d7e09d03 | 265 | LASSERT(lio->lis_nr_subios > 0); |
d7e09d03 PT |
266 | |
267 | stripe = lov_page_stripe(page); | |
0a3bdb00 | 268 | return lov_sub_get(env, lio, stripe); |
d7e09d03 PT |
269 | } |
270 | ||
d7e09d03 PT |
271 | static int lov_io_subio_init(const struct lu_env *env, struct lov_io *lio, |
272 | struct cl_io *io) | |
273 | { | |
0fcd869f | 274 | struct lov_stripe_md *lsm; |
d7e09d03 PT |
275 | int result; |
276 | ||
00697c43 | 277 | LASSERT(lio->lis_object); |
0fcd869f | 278 | lsm = lio->lis_object->lo_lsm; |
d7e09d03 PT |
279 | |
280 | /* | |
281 | * Need to be optimized, we can't afford to allocate a piece of memory | |
282 | * when writing a page. -jay | |
283 | */ | |
3d0ba716 JL |
284 | lio->lis_subs = |
285 | libcfs_kvzalloc(lsm->lsm_stripe_count * | |
286 | sizeof(lio->lis_subs[0]), | |
287 | GFP_NOFS); | |
00697c43 | 288 | if (lio->lis_subs) { |
d7e09d03 PT |
289 | lio->lis_nr_subios = lio->lis_stripe_count; |
290 | lio->lis_single_subio_index = -1; | |
291 | lio->lis_active_subios = 0; | |
292 | result = 0; | |
da5ecb4d | 293 | } else { |
d7e09d03 | 294 | result = -ENOMEM; |
da5ecb4d | 295 | } |
0a3bdb00 | 296 | return result; |
d7e09d03 PT |
297 | } |
298 | ||
865b734e FY |
299 | static int lov_io_slice_init(struct lov_io *lio, struct lov_object *obj, |
300 | struct cl_io *io) | |
d7e09d03 | 301 | { |
d7e09d03 PT |
302 | io->ci_result = 0; |
303 | lio->lis_object = obj; | |
304 | ||
d7e09d03 PT |
305 | lio->lis_stripe_count = obj->lo_lsm->lsm_stripe_count; |
306 | ||
307 | switch (io->ci_type) { | |
308 | case CIT_READ: | |
309 | case CIT_WRITE: | |
310 | lio->lis_pos = io->u.ci_rw.crw_pos; | |
311 | lio->lis_endpos = io->u.ci_rw.crw_pos + io->u.ci_rw.crw_count; | |
312 | lio->lis_io_endpos = lio->lis_endpos; | |
313 | if (cl_io_is_append(io)) { | |
314 | LASSERT(io->ci_type == CIT_WRITE); | |
865b734e FY |
315 | |
316 | /* | |
317 | * If there is LOV EA hole, then we may cannot locate | |
318 | * the current file-tail exactly. | |
319 | */ | |
320 | if (unlikely(obj->lo_lsm->lsm_pattern & | |
321 | LOV_PATTERN_F_HOLE)) | |
322 | return -EIO; | |
323 | ||
d7e09d03 PT |
324 | lio->lis_pos = 0; |
325 | lio->lis_endpos = OBD_OBJECT_EOF; | |
326 | } | |
327 | break; | |
328 | ||
329 | case CIT_SETATTR: | |
330 | if (cl_io_is_trunc(io)) | |
331 | lio->lis_pos = io->u.ci_setattr.sa_attr.lvb_size; | |
332 | else | |
333 | lio->lis_pos = 0; | |
334 | lio->lis_endpos = OBD_OBJECT_EOF; | |
335 | break; | |
336 | ||
337 | case CIT_FAULT: { | |
338 | pgoff_t index = io->u.ci_fault.ft_index; | |
50ffcb7e | 339 | |
d7e09d03 PT |
340 | lio->lis_pos = cl_offset(io->ci_obj, index); |
341 | lio->lis_endpos = cl_offset(io->ci_obj, index + 1); | |
342 | break; | |
343 | } | |
344 | ||
345 | case CIT_FSYNC: { | |
346 | lio->lis_pos = io->u.ci_fsync.fi_start; | |
347 | lio->lis_endpos = io->u.ci_fsync.fi_end; | |
348 | break; | |
349 | } | |
350 | ||
351 | case CIT_MISC: | |
352 | lio->lis_pos = 0; | |
353 | lio->lis_endpos = OBD_OBJECT_EOF; | |
354 | break; | |
355 | ||
356 | default: | |
357 | LBUG(); | |
358 | } | |
865b734e | 359 | return 0; |
d7e09d03 PT |
360 | } |
361 | ||
362 | static void lov_io_fini(const struct lu_env *env, const struct cl_io_slice *ios) | |
363 | { | |
364 | struct lov_io *lio = cl2lov_io(env, ios); | |
365 | struct lov_object *lov = cl2lov(ios->cis_obj); | |
366 | int i; | |
367 | ||
00697c43 | 368 | if (lio->lis_subs) { |
d7e09d03 PT |
369 | for (i = 0; i < lio->lis_nr_subios; i++) |
370 | lov_io_sub_fini(env, lio, &lio->lis_subs[i]); | |
3d0ba716 | 371 | kvfree(lio->lis_subs); |
d7e09d03 PT |
372 | lio->lis_nr_subios = 0; |
373 | } | |
374 | ||
375 | LASSERT(atomic_read(&lov->lo_active_ios) > 0); | |
376 | if (atomic_dec_and_test(&lov->lo_active_ios)) | |
377 | wake_up_all(&lov->lo_waitq); | |
d7e09d03 PT |
378 | } |
379 | ||
21aef7d9 | 380 | static u64 lov_offset_mod(u64 val, int delta) |
d7e09d03 PT |
381 | { |
382 | if (val != OBD_OBJECT_EOF) | |
383 | val += delta; | |
384 | return val; | |
385 | } | |
386 | ||
387 | static int lov_io_iter_init(const struct lu_env *env, | |
388 | const struct cl_io_slice *ios) | |
389 | { | |
390 | struct lov_io *lio = cl2lov_io(env, ios); | |
391 | struct lov_stripe_md *lsm = lio->lis_object->lo_lsm; | |
392 | struct lov_io_sub *sub; | |
21aef7d9 OD |
393 | u64 endpos; |
394 | u64 start; | |
395 | u64 end; | |
d7e09d03 PT |
396 | int stripe; |
397 | int rc = 0; | |
398 | ||
d7e09d03 PT |
399 | endpos = lov_offset_mod(lio->lis_endpos, -1); |
400 | for (stripe = 0; stripe < lio->lis_stripe_count; stripe++) { | |
401 | if (!lov_stripe_intersects(lsm, stripe, lio->lis_pos, | |
402 | endpos, &start, &end)) | |
403 | continue; | |
404 | ||
00697c43 | 405 | if (unlikely(!lov_r0(lio->lis_object)->lo_sub[stripe])) { |
397632e4 YS |
406 | if (ios->cis_io->ci_type == CIT_READ || |
407 | ios->cis_io->ci_type == CIT_WRITE || | |
408 | ios->cis_io->ci_type == CIT_FAULT) | |
409 | return -EIO; | |
410 | ||
411 | continue; | |
412 | } | |
413 | ||
822aa3e2 | 414 | end = lov_offset_mod(end, 1); |
d7e09d03 PT |
415 | sub = lov_sub_get(env, lio, stripe); |
416 | if (!IS_ERR(sub)) { | |
417 | lov_io_sub_inherit(sub->sub_io, lio, stripe, | |
418 | start, end); | |
419 | rc = cl_io_iter_init(sub->sub_env, sub->sub_io); | |
420 | lov_sub_put(sub); | |
b0f5aad5 | 421 | CDEBUG(D_VFSTRACE, "shrink: %d [%llu, %llu)\n", |
d7e09d03 | 422 | stripe, start, end); |
da5ecb4d | 423 | } else { |
d7e09d03 | 424 | rc = PTR_ERR(sub); |
da5ecb4d | 425 | } |
d7e09d03 PT |
426 | |
427 | if (!rc) | |
428 | list_add_tail(&sub->sub_linkage, &lio->lis_active); | |
429 | else | |
430 | break; | |
431 | } | |
0a3bdb00 | 432 | return rc; |
d7e09d03 PT |
433 | } |
434 | ||
435 | static int lov_io_rw_iter_init(const struct lu_env *env, | |
436 | const struct cl_io_slice *ios) | |
437 | { | |
438 | struct lov_io *lio = cl2lov_io(env, ios); | |
439 | struct cl_io *io = ios->cis_io; | |
440 | struct lov_stripe_md *lsm = lio->lis_object->lo_lsm; | |
b69835a2 | 441 | __u64 start = io->u.ci_rw.crw_pos; |
d7e09d03 PT |
442 | loff_t next; |
443 | unsigned long ssize = lsm->lsm_stripe_size; | |
444 | ||
445 | LASSERT(io->ci_type == CIT_READ || io->ci_type == CIT_WRITE); | |
d7e09d03 PT |
446 | |
447 | /* fast path for common case. */ | |
448 | if (lio->lis_nr_subios != 1 && !cl_io_is_append(io)) { | |
d7e09d03 PT |
449 | lov_do_div64(start, ssize); |
450 | next = (start + 1) * ssize; | |
451 | if (next <= start * ssize) | |
452 | next = ~0ull; | |
453 | ||
454 | io->ci_continue = next < lio->lis_io_endpos; | |
455 | io->u.ci_rw.crw_count = min_t(loff_t, lio->lis_io_endpos, | |
456 | next) - io->u.ci_rw.crw_pos; | |
457 | lio->lis_pos = io->u.ci_rw.crw_pos; | |
458 | lio->lis_endpos = io->u.ci_rw.crw_pos + io->u.ci_rw.crw_count; | |
b0f5aad5 GKH |
459 | CDEBUG(D_VFSTRACE, "stripe: %llu chunk: [%llu, %llu) %llu\n", |
460 | (__u64)start, lio->lis_pos, lio->lis_endpos, | |
d7e09d03 PT |
461 | (__u64)lio->lis_io_endpos); |
462 | } | |
463 | /* | |
464 | * XXX The following call should be optimized: we know, that | |
465 | * [lio->lis_pos, lio->lis_endpos) intersects with exactly one stripe. | |
466 | */ | |
0a3bdb00 | 467 | return lov_io_iter_init(env, ios); |
d7e09d03 PT |
468 | } |
469 | ||
470 | static int lov_io_call(const struct lu_env *env, struct lov_io *lio, | |
471 | int (*iofunc)(const struct lu_env *, struct cl_io *)) | |
472 | { | |
473 | struct cl_io *parent = lio->lis_cl.cis_io; | |
474 | struct lov_io_sub *sub; | |
475 | int rc = 0; | |
476 | ||
d7e09d03 PT |
477 | list_for_each_entry(sub, &lio->lis_active, sub_linkage) { |
478 | lov_sub_enter(sub); | |
479 | rc = iofunc(sub->sub_env, sub->sub_io); | |
480 | lov_sub_exit(sub); | |
481 | if (rc) | |
482 | break; | |
483 | ||
484 | if (parent->ci_result == 0) | |
485 | parent->ci_result = sub->sub_io->ci_result; | |
486 | } | |
0a3bdb00 | 487 | return rc; |
d7e09d03 PT |
488 | } |
489 | ||
490 | static int lov_io_lock(const struct lu_env *env, const struct cl_io_slice *ios) | |
491 | { | |
0a3bdb00 | 492 | return lov_io_call(env, cl2lov_io(env, ios), cl_io_lock); |
d7e09d03 PT |
493 | } |
494 | ||
495 | static int lov_io_start(const struct lu_env *env, const struct cl_io_slice *ios) | |
496 | { | |
0a3bdb00 | 497 | return lov_io_call(env, cl2lov_io(env, ios), cl_io_start); |
d7e09d03 PT |
498 | } |
499 | ||
500 | static int lov_io_end_wrapper(const struct lu_env *env, struct cl_io *io) | |
501 | { | |
d7e09d03 PT |
502 | /* |
503 | * It's possible that lov_io_start() wasn't called against this | |
504 | * sub-io, either because previous sub-io failed, or upper layer | |
505 | * completed IO. | |
506 | */ | |
507 | if (io->ci_state == CIS_IO_GOING) | |
508 | cl_io_end(env, io); | |
509 | else | |
510 | io->ci_state = CIS_IO_FINISHED; | |
0a3bdb00 | 511 | return 0; |
d7e09d03 PT |
512 | } |
513 | ||
514 | static int lov_io_iter_fini_wrapper(const struct lu_env *env, struct cl_io *io) | |
515 | { | |
516 | cl_io_iter_fini(env, io); | |
0a3bdb00 | 517 | return 0; |
d7e09d03 PT |
518 | } |
519 | ||
520 | static int lov_io_unlock_wrapper(const struct lu_env *env, struct cl_io *io) | |
521 | { | |
522 | cl_io_unlock(env, io); | |
0a3bdb00 | 523 | return 0; |
d7e09d03 PT |
524 | } |
525 | ||
526 | static void lov_io_end(const struct lu_env *env, const struct cl_io_slice *ios) | |
527 | { | |
528 | int rc; | |
529 | ||
530 | rc = lov_io_call(env, cl2lov_io(env, ios), lov_io_end_wrapper); | |
531 | LASSERT(rc == 0); | |
532 | } | |
533 | ||
534 | static void lov_io_iter_fini(const struct lu_env *env, | |
535 | const struct cl_io_slice *ios) | |
536 | { | |
537 | struct lov_io *lio = cl2lov_io(env, ios); | |
538 | int rc; | |
539 | ||
d7e09d03 PT |
540 | rc = lov_io_call(env, lio, lov_io_iter_fini_wrapper); |
541 | LASSERT(rc == 0); | |
542 | while (!list_empty(&lio->lis_active)) | |
543 | list_del_init(lio->lis_active.next); | |
d7e09d03 PT |
544 | } |
545 | ||
546 | static void lov_io_unlock(const struct lu_env *env, | |
547 | const struct cl_io_slice *ios) | |
548 | { | |
549 | int rc; | |
550 | ||
d7e09d03 PT |
551 | rc = lov_io_call(env, cl2lov_io(env, ios), lov_io_unlock_wrapper); |
552 | LASSERT(rc == 0); | |
d7e09d03 PT |
553 | } |
554 | ||
d7e09d03 PT |
555 | /** |
556 | * lov implementation of cl_operations::cio_submit() method. It takes a list | |
557 | * of pages in \a queue, splits it into per-stripe sub-lists, invokes | |
558 | * cl_io_submit() on underlying devices to submit sub-lists, and then splices | |
559 | * everything back. | |
560 | * | |
561 | * Major complication of this function is a need to handle memory cleansing: | |
562 | * cl_io_submit() is called to write out pages as a part of VM memory | |
563 | * reclamation, and hence it may not fail due to memory shortages (system | |
564 | * dead-locks otherwise). To deal with this, some resources (sub-lists, | |
565 | * sub-environment, etc.) are allocated per-device on "startup" (i.e., in a | |
566 | * not-memory cleansing context), and in case of memory shortage, these | |
567 | * pre-allocated resources are used by lov_io_submit() under | |
568 | * lov_device::ld_mutex mutex. | |
569 | */ | |
570 | static int lov_io_submit(const struct lu_env *env, | |
571 | const struct cl_io_slice *ios, | |
572 | enum cl_req_type crt, struct cl_2queue *queue) | |
573 | { | |
77605e41 JX |
574 | struct cl_page_list *qin = &queue->c2_qin; |
575 | struct lov_io *lio = cl2lov_io(env, ios); | |
576 | struct lov_io_sub *sub; | |
577 | struct cl_page_list *plist = &lov_env_info(env)->lti_plist; | |
d7e09d03 | 578 | struct cl_page *page; |
d7e09d03 PT |
579 | int stripe; |
580 | ||
d7e09d03 | 581 | int rc = 0; |
29aaf496 | 582 | |
d7e09d03 PT |
583 | if (lio->lis_active_subios == 1) { |
584 | int idx = lio->lis_single_subio_index; | |
d7e09d03 PT |
585 | |
586 | LASSERT(idx < lio->lis_nr_subios); | |
587 | sub = lov_sub_get(env, lio, idx); | |
588 | LASSERT(!IS_ERR(sub)); | |
589 | LASSERT(sub->sub_io == &lio->lis_single_subio); | |
590 | rc = cl_io_submit_rw(sub->sub_env, sub->sub_io, | |
591 | crt, queue); | |
592 | lov_sub_put(sub); | |
0a3bdb00 | 593 | return rc; |
d7e09d03 PT |
594 | } |
595 | ||
00697c43 | 596 | LASSERT(lio->lis_subs); |
d7e09d03 | 597 | |
77605e41 JX |
598 | cl_page_list_init(plist); |
599 | while (qin->pl_nr > 0) { | |
600 | struct cl_2queue *cl2q = &lov_env_info(env)->lti_cl2q; | |
d7e09d03 | 601 | |
77605e41 | 602 | cl_2queue_init(cl2q); |
d7e09d03 | 603 | |
77605e41 JX |
604 | page = cl_page_list_first(qin); |
605 | cl_page_list_move(&cl2q->c2_qin, qin, page); | |
606 | ||
607 | stripe = lov_page_stripe(page); | |
608 | while (qin->pl_nr > 0) { | |
609 | page = cl_page_list_first(qin); | |
610 | if (stripe != lov_page_stripe(page)) | |
611 | break; | |
612 | ||
613 | cl_page_list_move(&cl2q->c2_qin, qin, page); | |
614 | } | |
d7e09d03 | 615 | |
d7e09d03 PT |
616 | sub = lov_sub_get(env, lio, stripe); |
617 | if (!IS_ERR(sub)) { | |
618 | rc = cl_io_submit_rw(sub->sub_env, sub->sub_io, | |
619 | crt, cl2q); | |
620 | lov_sub_put(sub); | |
77605e41 | 621 | } else { |
d7e09d03 | 622 | rc = PTR_ERR(sub); |
77605e41 JX |
623 | } |
624 | ||
625 | cl_page_list_splice(&cl2q->c2_qin, plist); | |
d7e09d03 | 626 | cl_page_list_splice(&cl2q->c2_qout, &queue->c2_qout); |
77605e41 JX |
627 | cl_2queue_fini(env, cl2q); |
628 | ||
d7e09d03 PT |
629 | if (rc != 0) |
630 | break; | |
631 | } | |
632 | ||
77605e41 JX |
633 | cl_page_list_splice(plist, qin); |
634 | cl_page_list_fini(env, plist); | |
d7e09d03 | 635 | |
77605e41 JX |
636 | return rc; |
637 | } | |
638 | ||
639 | static int lov_io_commit_async(const struct lu_env *env, | |
640 | const struct cl_io_slice *ios, | |
641 | struct cl_page_list *queue, int from, int to, | |
642 | cl_commit_cbt cb) | |
643 | { | |
644 | struct cl_page_list *plist = &lov_env_info(env)->lti_plist; | |
645 | struct lov_io *lio = cl2lov_io(env, ios); | |
646 | struct lov_io_sub *sub; | |
647 | struct cl_page *page; | |
648 | int rc = 0; | |
649 | ||
650 | if (lio->lis_active_subios == 1) { | |
651 | int idx = lio->lis_single_subio_index; | |
d7e09d03 | 652 | |
77605e41 JX |
653 | LASSERT(idx < lio->lis_nr_subios); |
654 | sub = lov_sub_get(env, lio, idx); | |
655 | LASSERT(!IS_ERR(sub)); | |
656 | LASSERT(sub->sub_io == &lio->lis_single_subio); | |
657 | rc = cl_io_commit_async(sub->sub_env, sub->sub_io, queue, | |
658 | from, to, cb); | |
659 | lov_sub_put(sub); | |
660 | return rc; | |
d7e09d03 PT |
661 | } |
662 | ||
77605e41 JX |
663 | LASSERT(lio->lis_subs); |
664 | ||
665 | cl_page_list_init(plist); | |
666 | while (queue->pl_nr > 0) { | |
667 | int stripe_to = to; | |
668 | int stripe; | |
d7e09d03 | 669 | |
77605e41 JX |
670 | LASSERT(plist->pl_nr == 0); |
671 | page = cl_page_list_first(queue); | |
672 | cl_page_list_move(plist, queue, page); | |
d7e09d03 | 673 | |
77605e41 JX |
674 | stripe = lov_page_stripe(page); |
675 | while (queue->pl_nr > 0) { | |
676 | page = cl_page_list_first(queue); | |
677 | if (stripe != lov_page_stripe(page)) | |
678 | break; | |
679 | ||
680 | cl_page_list_move(plist, queue, page); | |
d7e09d03 | 681 | } |
d7e09d03 | 682 | |
77605e41 JX |
683 | if (queue->pl_nr > 0) /* still has more pages */ |
684 | stripe_to = PAGE_SIZE; | |
d7e09d03 | 685 | |
77605e41 JX |
686 | sub = lov_sub_get(env, lio, stripe); |
687 | if (!IS_ERR(sub)) { | |
688 | rc = cl_io_commit_async(sub->sub_env, sub->sub_io, | |
689 | plist, from, stripe_to, cb); | |
690 | lov_sub_put(sub); | |
691 | } else { | |
692 | rc = PTR_ERR(sub); | |
693 | break; | |
694 | } | |
d7e09d03 | 695 | |
77605e41 JX |
696 | if (plist->pl_nr > 0) /* short write */ |
697 | break; | |
d7e09d03 | 698 | |
77605e41 JX |
699 | from = 0; |
700 | } | |
d7e09d03 | 701 | |
77605e41 JX |
702 | /* for error case, add the page back into the qin list */ |
703 | LASSERT(ergo(rc == 0, plist->pl_nr == 0)); | |
704 | while (plist->pl_nr > 0) { | |
705 | /* error occurred, add the uncommitted pages back into queue */ | |
706 | page = cl_page_list_last(plist); | |
707 | cl_page_list_move_head(queue, plist, page); | |
708 | } | |
709 | ||
710 | return rc; | |
d7e09d03 PT |
711 | } |
712 | ||
713 | static int lov_io_fault_start(const struct lu_env *env, | |
714 | const struct cl_io_slice *ios) | |
715 | { | |
716 | struct cl_fault_io *fio; | |
717 | struct lov_io *lio; | |
718 | struct lov_io_sub *sub; | |
719 | ||
d7e09d03 PT |
720 | fio = &ios->cis_io->u.ci_fault; |
721 | lio = cl2lov_io(env, ios); | |
722 | sub = lov_sub_get(env, lio, lov_page_stripe(fio->ft_page)); | |
d0f69b5a SM |
723 | if (IS_ERR(sub)) |
724 | return PTR_ERR(sub); | |
d7e09d03 PT |
725 | sub->sub_io->u.ci_fault.ft_nob = fio->ft_nob; |
726 | lov_sub_put(sub); | |
0a3bdb00 | 727 | return lov_io_start(env, ios); |
d7e09d03 PT |
728 | } |
729 | ||
730 | static void lov_io_fsync_end(const struct lu_env *env, | |
731 | const struct cl_io_slice *ios) | |
732 | { | |
733 | struct lov_io *lio = cl2lov_io(env, ios); | |
734 | struct lov_io_sub *sub; | |
735 | unsigned int *written = &ios->cis_io->u.ci_fsync.fi_nr_written; | |
d7e09d03 PT |
736 | |
737 | *written = 0; | |
738 | list_for_each_entry(sub, &lio->lis_active, sub_linkage) { | |
739 | struct cl_io *subio = sub->sub_io; | |
740 | ||
741 | lov_sub_enter(sub); | |
742 | lov_io_end_wrapper(sub->sub_env, subio); | |
743 | lov_sub_exit(sub); | |
744 | ||
745 | if (subio->ci_result == 0) | |
746 | *written += subio->u.ci_fsync.fi_nr_written; | |
747 | } | |
d7e09d03 PT |
748 | } |
749 | ||
750 | static const struct cl_io_operations lov_io_ops = { | |
751 | .op = { | |
752 | [CIT_READ] = { | |
753 | .cio_fini = lov_io_fini, | |
754 | .cio_iter_init = lov_io_rw_iter_init, | |
755 | .cio_iter_fini = lov_io_iter_fini, | |
756 | .cio_lock = lov_io_lock, | |
757 | .cio_unlock = lov_io_unlock, | |
758 | .cio_start = lov_io_start, | |
759 | .cio_end = lov_io_end | |
760 | }, | |
761 | [CIT_WRITE] = { | |
762 | .cio_fini = lov_io_fini, | |
763 | .cio_iter_init = lov_io_rw_iter_init, | |
764 | .cio_iter_fini = lov_io_iter_fini, | |
765 | .cio_lock = lov_io_lock, | |
766 | .cio_unlock = lov_io_unlock, | |
767 | .cio_start = lov_io_start, | |
768 | .cio_end = lov_io_end | |
769 | }, | |
770 | [CIT_SETATTR] = { | |
771 | .cio_fini = lov_io_fini, | |
772 | .cio_iter_init = lov_io_iter_init, | |
773 | .cio_iter_fini = lov_io_iter_fini, | |
774 | .cio_lock = lov_io_lock, | |
775 | .cio_unlock = lov_io_unlock, | |
776 | .cio_start = lov_io_start, | |
777 | .cio_end = lov_io_end | |
778 | }, | |
779 | [CIT_FAULT] = { | |
780 | .cio_fini = lov_io_fini, | |
781 | .cio_iter_init = lov_io_iter_init, | |
782 | .cio_iter_fini = lov_io_iter_fini, | |
783 | .cio_lock = lov_io_lock, | |
784 | .cio_unlock = lov_io_unlock, | |
785 | .cio_start = lov_io_fault_start, | |
786 | .cio_end = lov_io_end | |
787 | }, | |
788 | [CIT_FSYNC] = { | |
789 | .cio_fini = lov_io_fini, | |
790 | .cio_iter_init = lov_io_iter_init, | |
791 | .cio_iter_fini = lov_io_iter_fini, | |
792 | .cio_lock = lov_io_lock, | |
793 | .cio_unlock = lov_io_unlock, | |
794 | .cio_start = lov_io_start, | |
795 | .cio_end = lov_io_fsync_end | |
796 | }, | |
797 | [CIT_MISC] = { | |
798 | .cio_fini = lov_io_fini | |
799 | } | |
800 | }, | |
77605e41 JX |
801 | .cio_submit = lov_io_submit, |
802 | .cio_commit_async = lov_io_commit_async, | |
d7e09d03 PT |
803 | }; |
804 | ||
805 | /***************************************************************************** | |
806 | * | |
807 | * Empty lov io operations. | |
808 | * | |
809 | */ | |
810 | ||
811 | static void lov_empty_io_fini(const struct lu_env *env, | |
812 | const struct cl_io_slice *ios) | |
813 | { | |
814 | struct lov_object *lov = cl2lov(ios->cis_obj); | |
d7e09d03 PT |
815 | |
816 | if (atomic_dec_and_test(&lov->lo_active_ios)) | |
817 | wake_up_all(&lov->lo_waitq); | |
d7e09d03 PT |
818 | } |
819 | ||
820 | static void lov_empty_impossible(const struct lu_env *env, | |
821 | struct cl_io_slice *ios) | |
822 | { | |
823 | LBUG(); | |
824 | } | |
825 | ||
826 | #define LOV_EMPTY_IMPOSSIBLE ((void *)lov_empty_impossible) | |
827 | ||
828 | /** | |
829 | * An io operation vector for files without stripes. | |
830 | */ | |
831 | static const struct cl_io_operations lov_empty_io_ops = { | |
832 | .op = { | |
833 | [CIT_READ] = { | |
834 | .cio_fini = lov_empty_io_fini, | |
835 | #if 0 | |
836 | .cio_iter_init = LOV_EMPTY_IMPOSSIBLE, | |
837 | .cio_lock = LOV_EMPTY_IMPOSSIBLE, | |
838 | .cio_start = LOV_EMPTY_IMPOSSIBLE, | |
839 | .cio_end = LOV_EMPTY_IMPOSSIBLE | |
840 | #endif | |
841 | }, | |
842 | [CIT_WRITE] = { | |
843 | .cio_fini = lov_empty_io_fini, | |
844 | .cio_iter_init = LOV_EMPTY_IMPOSSIBLE, | |
845 | .cio_lock = LOV_EMPTY_IMPOSSIBLE, | |
846 | .cio_start = LOV_EMPTY_IMPOSSIBLE, | |
847 | .cio_end = LOV_EMPTY_IMPOSSIBLE | |
848 | }, | |
849 | [CIT_SETATTR] = { | |
850 | .cio_fini = lov_empty_io_fini, | |
851 | .cio_iter_init = LOV_EMPTY_IMPOSSIBLE, | |
852 | .cio_lock = LOV_EMPTY_IMPOSSIBLE, | |
853 | .cio_start = LOV_EMPTY_IMPOSSIBLE, | |
854 | .cio_end = LOV_EMPTY_IMPOSSIBLE | |
855 | }, | |
856 | [CIT_FAULT] = { | |
857 | .cio_fini = lov_empty_io_fini, | |
858 | .cio_iter_init = LOV_EMPTY_IMPOSSIBLE, | |
859 | .cio_lock = LOV_EMPTY_IMPOSSIBLE, | |
860 | .cio_start = LOV_EMPTY_IMPOSSIBLE, | |
861 | .cio_end = LOV_EMPTY_IMPOSSIBLE | |
862 | }, | |
863 | [CIT_FSYNC] = { | |
864 | .cio_fini = lov_empty_io_fini | |
865 | }, | |
866 | [CIT_MISC] = { | |
867 | .cio_fini = lov_empty_io_fini | |
868 | } | |
869 | }, | |
77605e41 JX |
870 | .cio_submit = LOV_EMPTY_IMPOSSIBLE, |
871 | .cio_commit_async = LOV_EMPTY_IMPOSSIBLE | |
d7e09d03 PT |
872 | }; |
873 | ||
874 | int lov_io_init_raid0(const struct lu_env *env, struct cl_object *obj, | |
875 | struct cl_io *io) | |
876 | { | |
877 | struct lov_io *lio = lov_env_io(env); | |
878 | struct lov_object *lov = cl2lov(obj); | |
879 | ||
d7e09d03 | 880 | INIT_LIST_HEAD(&lio->lis_active); |
865b734e | 881 | io->ci_result = lov_io_slice_init(lio, lov, io); |
d7e09d03 PT |
882 | if (io->ci_result == 0) { |
883 | io->ci_result = lov_io_subio_init(env, lio, io); | |
884 | if (io->ci_result == 0) { | |
885 | cl_io_slice_add(io, &lio->lis_cl, obj, &lov_io_ops); | |
886 | atomic_inc(&lov->lo_active_ios); | |
887 | } | |
888 | } | |
0a3bdb00 | 889 | return io->ci_result; |
d7e09d03 PT |
890 | } |
891 | ||
892 | int lov_io_init_empty(const struct lu_env *env, struct cl_object *obj, | |
893 | struct cl_io *io) | |
894 | { | |
895 | struct lov_object *lov = cl2lov(obj); | |
896 | struct lov_io *lio = lov_env_io(env); | |
897 | int result; | |
d7e09d03 PT |
898 | |
899 | lio->lis_object = lov; | |
900 | switch (io->ci_type) { | |
901 | default: | |
902 | LBUG(); | |
903 | case CIT_MISC: | |
904 | case CIT_READ: | |
905 | result = 0; | |
906 | break; | |
907 | case CIT_FSYNC: | |
908 | case CIT_SETATTR: | |
822aa3e2 | 909 | result = 1; |
d7e09d03 PT |
910 | break; |
911 | case CIT_WRITE: | |
912 | result = -EBADF; | |
913 | break; | |
914 | case CIT_FAULT: | |
915 | result = -EFAULT; | |
916 | CERROR("Page fault on a file without stripes: "DFID"\n", | |
917 | PFID(lu_object_fid(&obj->co_lu))); | |
918 | break; | |
919 | } | |
920 | if (result == 0) { | |
921 | cl_io_slice_add(io, &lio->lis_cl, obj, &lov_empty_io_ops); | |
922 | atomic_inc(&lov->lo_active_ios); | |
923 | } | |
924 | ||
925 | io->ci_result = result < 0 ? result : 0; | |
30889c72 | 926 | return result; |
d7e09d03 PT |
927 | } |
928 | ||
5dd16419 | 929 | int lov_io_init_released(const struct lu_env *env, struct cl_object *obj, |
f1564f16 | 930 | struct cl_io *io) |
5dd16419 JX |
931 | { |
932 | struct lov_object *lov = cl2lov(obj); | |
933 | struct lov_io *lio = lov_env_io(env); | |
934 | int result; | |
5dd16419 | 935 | |
00697c43 | 936 | LASSERT(lov->lo_lsm); |
5dd16419 JX |
937 | lio->lis_object = lov; |
938 | ||
939 | switch (io->ci_type) { | |
940 | default: | |
941 | LASSERTF(0, "invalid type %d\n", io->ci_type); | |
942 | case CIT_MISC: | |
943 | case CIT_FSYNC: | |
5ea17d6c | 944 | result = 1; |
5dd16419 | 945 | break; |
251c4317 | 946 | case CIT_SETATTR: |
5ea17d6c JL |
947 | /* the truncate to 0 is managed by MDT: |
948 | * - in open, for open O_TRUNC | |
949 | * - in setattr, for truncate | |
950 | */ | |
951 | /* the truncate is for size > 0 so triggers a restore */ | |
952 | if (cl_io_is_trunc(io)) | |
953 | io->ci_restore_needed = 1; | |
954 | result = -ENODATA; | |
955 | break; | |
5dd16419 JX |
956 | case CIT_READ: |
957 | case CIT_WRITE: | |
958 | case CIT_FAULT: | |
5ea17d6c JL |
959 | io->ci_restore_needed = 1; |
960 | result = -ENODATA; | |
5dd16419 JX |
961 | break; |
962 | } | |
963 | if (result == 0) { | |
964 | cl_io_slice_add(io, &lio->lis_cl, obj, &lov_empty_io_ops); | |
965 | atomic_inc(&lov->lo_active_ios); | |
966 | } | |
967 | ||
968 | io->ci_result = result < 0 ? result : 0; | |
30889c72 | 969 | return result; |
5dd16419 | 970 | } |
c9f6bb96 | 971 | |
d7e09d03 | 972 | /** @} lov */ |