projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
f2fs: speed up handling holes in fiemap
[deliverable/linux.git] / fs / f2fs / data.c
1 /*
2 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11 #include <linux/fs.h>
12 #include <linux/f2fs_fs.h>
13 #include <linux/buffer_head.h>
14 #include <linux/mpage.h>
15 #include <linux/writeback.h>
16 #include <linux/backing-dev.h>
17 #include <linux/pagevec.h>
18 #include <linux/blkdev.h>
19 #include <linux/bio.h>
20 #include <linux/prefetch.h>
21 #include <linux/uio.h>
22 #include <linux/cleancache.h>
23
24 #include "f2fs.h"
25 #include "node.h"
26 #include "segment.h"
27 #include "trace.h"
28 #include <trace/events/f2fs.h>
29
30 static void f2fs_read_end_io(struct bio *bio)
31 {
32 struct bio_vec *bvec;
33 int i;
34
35 if (f2fs_bio_encrypted(bio)) {
36 if (bio->bi_error) {
37 f2fs_release_crypto_ctx(bio->bi_private);
38 } else {
39 f2fs_end_io_crypto_work(bio->bi_private, bio);
40 return;
41 }
42 }
43
44 bio_for_each_segment_all(bvec, bio, i) {
45 struct page *page = bvec->bv_page;
46
47 if (!bio->bi_error) {
48 SetPageUptodate(page);
49 } else {
50 ClearPageUptodate(page);
51 SetPageError(page);
52 }
53 unlock_page(page);
54 }
55 bio_put(bio);
56 }
57
58 static void f2fs_write_end_io(struct bio *bio)
59 {
60 struct f2fs_sb_info *sbi = bio->bi_private;
61 struct bio_vec *bvec;
62 int i;
63
64 bio_for_each_segment_all(bvec, bio, i) {
65 struct page *page = bvec->bv_page;
66
67 f2fs_restore_and_release_control_page(&page);
68
69 if (unlikely(bio->bi_error)) {
70 set_bit(AS_EIO, &page->mapping->flags);
71 f2fs_stop_checkpoint(sbi);
72 }
73 end_page_writeback(page);
74 dec_page_count(sbi, F2FS_WRITEBACK);
75 }
76
77 if (!get_pages(sbi, F2FS_WRITEBACK) && wq_has_sleeper(&sbi->cp_wait))
78 wake_up(&sbi->cp_wait);
79
80 bio_put(bio);
81 }
82
83 /*
84 * Low-level block read/write IO operations.
85 */
86 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
87 int npages, bool is_read)
88 {
89 struct bio *bio;
90
91 bio = f2fs_bio_alloc(npages);
92
93 bio->bi_bdev = sbi->sb->s_bdev;
94 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
95 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
96 bio->bi_private = is_read ? NULL : sbi;
97
98 return bio;
99 }
100
101 static void __submit_merged_bio(struct f2fs_bio_info *io)
102 {
103 struct f2fs_io_info *fio = &io->fio;
104
105 if (!io->bio)
106 return;
107
108 if (is_read_io(fio->rw))
109 trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio);
110 else
111 trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio);
112
113 submit_bio(fio->rw, io->bio);
114 io->bio = NULL;
115 }
116
117 bool is_merged_page(struct f2fs_sb_info *sbi, struct page *page,
118 enum page_type type)
119 {
120 enum page_type btype = PAGE_TYPE_OF_BIO(type);
121 struct f2fs_bio_info *io = &sbi->write_io[btype];
122 struct bio_vec *bvec;
123 struct page *target;
124 int i;
125
126 down_read(&io->io_rwsem);
127 if (!io->bio) {
128 up_read(&io->io_rwsem);
129 return false;
130 }
131
132 bio_for_each_segment_all(bvec, io->bio, i) {
133
134 if (bvec->bv_page->mapping) {
135 target = bvec->bv_page;
136 } else {
137 struct f2fs_crypto_ctx *ctx;
138
139 /* encrypted page */
140 ctx = (struct f2fs_crypto_ctx *)page_private(
141 bvec->bv_page);
142 target = ctx->w.control_page;
143 }
144
145 if (page == target) {
146 up_read(&io->io_rwsem);
147 return true;
148 }
149 }
150
151 up_read(&io->io_rwsem);
152 return false;
153 }
154
155 void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
156 enum page_type type, int rw)
157 {
158 enum page_type btype = PAGE_TYPE_OF_BIO(type);
159 struct f2fs_bio_info *io;
160
161 io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype];
162
163 down_write(&io->io_rwsem);
164
165 /* change META to META_FLUSH in the checkpoint procedure */
166 if (type >= META_FLUSH) {
167 io->fio.type = META_FLUSH;
168 if (test_opt(sbi, NOBARRIER))
169 io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO;
170 else
171 io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO;
172 }
173 __submit_merged_bio(io);
174 up_write(&io->io_rwsem);
175 }
176
177 /*
178 * Fill the locked page with data located in the block address.
179 * Return unlocked page.
180 */
181 int f2fs_submit_page_bio(struct f2fs_io_info *fio)
182 {
183 struct bio *bio;
184 struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page;
185
186 trace_f2fs_submit_page_bio(page, fio);
187 f2fs_trace_ios(fio, 0);
188
189 /* Allocate a new bio */
190 bio = __bio_alloc(fio->sbi, fio->blk_addr, 1, is_read_io(fio->rw));
191
192 if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
193 bio_put(bio);
194 return -EFAULT;
195 }
196
197 submit_bio(fio->rw, bio);
198 return 0;
199 }
200
201 void f2fs_submit_page_mbio(struct f2fs_io_info *fio)
202 {
203 struct f2fs_sb_info *sbi = fio->sbi;
204 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
205 struct f2fs_bio_info *io;
206 bool is_read = is_read_io(fio->rw);
207 struct page *bio_page;
208
209 io = is_read ? &sbi->read_io : &sbi->write_io[btype];
210
211 verify_block_addr(sbi, fio->blk_addr);
212
213 down_write(&io->io_rwsem);
214
215 if (!is_read)
216 inc_page_count(sbi, F2FS_WRITEBACK);
217
218 if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 ||
219 io->fio.rw != fio->rw))
220 __submit_merged_bio(io);
221 alloc_new:
222 if (io->bio == NULL) {
223 int bio_blocks = MAX_BIO_BLOCKS(sbi);
224
225 io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read);
226 io->fio = *fio;
227 }
228
229 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
230
231 if (bio_add_page(io->bio, bio_page, PAGE_CACHE_SIZE, 0) <
232 PAGE_CACHE_SIZE) {
233 __submit_merged_bio(io);
234 goto alloc_new;
235 }
236
237 io->last_block_in_bio = fio->blk_addr;
238 f2fs_trace_ios(fio, 0);
239
240 up_write(&io->io_rwsem);
241 trace_f2fs_submit_page_mbio(fio->page, fio);
242 }
243
244 /*
245 * Lock ordering for the change of data block address:
246 * ->data_page
247 * ->node_page
248 * update block addresses in the node page
249 */
250 void set_data_blkaddr(struct dnode_of_data *dn)
251 {
252 struct f2fs_node *rn;
253 __le32 *addr_array;
254 struct page *node_page = dn->node_page;
255 unsigned int ofs_in_node = dn->ofs_in_node;
256
257 f2fs_wait_on_page_writeback(node_page, NODE, true);
258
259 rn = F2FS_NODE(node_page);
260
261 /* Get physical address of data block */
262 addr_array = blkaddr_in_node(rn);
263 addr_array[ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
264 if (set_page_dirty(node_page))
265 dn->node_changed = true;
266 }
267
268 int reserve_new_block(struct dnode_of_data *dn)
269 {
270 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
271
272 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
273 return -EPERM;
274 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
275 return -ENOSPC;
276
277 trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);
278
279 dn->data_blkaddr = NEW_ADDR;
280 set_data_blkaddr(dn);
281 mark_inode_dirty(dn->inode);
282 sync_inode_page(dn);
283 return 0;
284 }
285
286 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
287 {
288 bool need_put = dn->inode_page ? false : true;
289 int err;
290
291 err = get_dnode_of_data(dn, index, ALLOC_NODE);
292 if (err)
293 return err;
294
295 if (dn->data_blkaddr == NULL_ADDR)
296 err = reserve_new_block(dn);
297 if (err || need_put)
298 f2fs_put_dnode(dn);
299 return err;
300 }
301
302 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
303 {
304 struct extent_info ei;
305 struct inode *inode = dn->inode;
306
307 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
308 dn->data_blkaddr = ei.blk + index - ei.fofs;
309 return 0;
310 }
311
312 return f2fs_reserve_block(dn, index);
313 }
314
315 struct page *get_read_data_page(struct inode *inode, pgoff_t index,
316 int rw, bool for_write)
317 {
318 struct address_space *mapping = inode->i_mapping;
319 struct dnode_of_data dn;
320 struct page *page;
321 struct extent_info ei;
322 int err;
323 struct f2fs_io_info fio = {
324 .sbi = F2FS_I_SB(inode),
325 .type = DATA,
326 .rw = rw,
327 .encrypted_page = NULL,
328 };
329
330 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
331 return read_mapping_page(mapping, index, NULL);
332
333 page = f2fs_grab_cache_page(mapping, index, for_write);
334 if (!page)
335 return ERR_PTR(-ENOMEM);
336
337 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
338 dn.data_blkaddr = ei.blk + index - ei.fofs;
339 goto got_it;
340 }
341
342 set_new_dnode(&dn, inode, NULL, NULL, 0);
343 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
344 if (err)
345 goto put_err;
346 f2fs_put_dnode(&dn);
347
348 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
349 err = -ENOENT;
350 goto put_err;
351 }
352 got_it:
353 if (PageUptodate(page)) {
354 unlock_page(page);
355 return page;
356 }
357
358 /*
359 * A new dentry page is allocated but not able to be written, since its
360 * new inode page couldn't be allocated due to -ENOSPC.
361 * In such the case, its blkaddr can be remained as NEW_ADDR.
362 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
363 */
364 if (dn.data_blkaddr == NEW_ADDR) {
365 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
366 SetPageUptodate(page);
367 unlock_page(page);
368 return page;
369 }
370
371 fio.blk_addr = dn.data_blkaddr;
372 fio.page = page;
373 err = f2fs_submit_page_bio(&fio);
374 if (err)
375 goto put_err;
376 return page;
377
378 put_err:
379 f2fs_put_page(page, 1);
380 return ERR_PTR(err);
381 }
382
383 struct page *find_data_page(struct inode *inode, pgoff_t index)
384 {
385 struct address_space *mapping = inode->i_mapping;
386 struct page *page;
387
388 page = find_get_page(mapping, index);
389 if (page && PageUptodate(page))
390 return page;
391 f2fs_put_page(page, 0);
392
393 page = get_read_data_page(inode, index, READ_SYNC, false);
394 if (IS_ERR(page))
395 return page;
396
397 if (PageUptodate(page))
398 return page;
399
400 wait_on_page_locked(page);
401 if (unlikely(!PageUptodate(page))) {
402 f2fs_put_page(page, 0);
403 return ERR_PTR(-EIO);
404 }
405 return page;
406 }
407
408 /*
409 * If it tries to access a hole, return an error.
410 * Because, the callers, functions in dir.c and GC, should be able to know
411 * whether this page exists or not.
412 */
413 struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
414 bool for_write)
415 {
416 struct address_space *mapping = inode->i_mapping;
417 struct page *page;
418 repeat:
419 page = get_read_data_page(inode, index, READ_SYNC, for_write);
420 if (IS_ERR(page))
421 return page;
422
423 /* wait for read completion */
424 lock_page(page);
425 if (unlikely(!PageUptodate(page))) {
426 f2fs_put_page(page, 1);
427 return ERR_PTR(-EIO);
428 }
429 if (unlikely(page->mapping != mapping)) {
430 f2fs_put_page(page, 1);
431 goto repeat;
432 }
433 return page;
434 }
435
436 /*
437 * Caller ensures that this data page is never allocated.
438 * A new zero-filled data page is allocated in the page cache.
439 *
440 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
441 * f2fs_unlock_op().
442 * Note that, ipage is set only by make_empty_dir, and if any error occur,
443 * ipage should be released by this function.
444 */
445 struct page *get_new_data_page(struct inode *inode,
446 struct page *ipage, pgoff_t index, bool new_i_size)
447 {
448 struct address_space *mapping = inode->i_mapping;
449 struct page *page;
450 struct dnode_of_data dn;
451 int err;
452
453 page = f2fs_grab_cache_page(mapping, index, true);
454 if (!page) {
455 /*
456 * before exiting, we should make sure ipage will be released
457 * if any error occur.
458 */
459 f2fs_put_page(ipage, 1);
460 return ERR_PTR(-ENOMEM);
461 }
462
463 set_new_dnode(&dn, inode, ipage, NULL, 0);
464 err = f2fs_reserve_block(&dn, index);
465 if (err) {
466 f2fs_put_page(page, 1);
467 return ERR_PTR(err);
468 }
469 if (!ipage)
470 f2fs_put_dnode(&dn);
471
472 if (PageUptodate(page))
473 goto got_it;
474
475 if (dn.data_blkaddr == NEW_ADDR) {
476 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
477 SetPageUptodate(page);
478 } else {
479 f2fs_put_page(page, 1);
480
481 /* if ipage exists, blkaddr should be NEW_ADDR */
482 f2fs_bug_on(F2FS_I_SB(inode), ipage);
483 page = get_lock_data_page(inode, index, true);
484 if (IS_ERR(page))
485 return page;
486 }
487 got_it:
488 if (new_i_size && i_size_read(inode) <
489 ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)) {
490 i_size_write(inode, ((loff_t)(index + 1) << PAGE_CACHE_SHIFT));
491 /* Only the directory inode sets new_i_size */
492 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
493 }
494 return page;
495 }
496
497 static int __allocate_data_block(struct dnode_of_data *dn)
498 {
499 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
500 struct f2fs_summary sum;
501 struct node_info ni;
502 int seg = CURSEG_WARM_DATA;
503 pgoff_t fofs;
504
505 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
506 return -EPERM;
507
508 dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
509 if (dn->data_blkaddr == NEW_ADDR)
510 goto alloc;
511
512 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
513 return -ENOSPC;
514
515 alloc:
516 get_node_info(sbi, dn->nid, &ni);
517 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
518
519 if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page)
520 seg = CURSEG_DIRECT_IO;
521
522 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
523 &sum, seg);
524 set_data_blkaddr(dn);
525
526 /* update i_size */
527 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
528 dn->ofs_in_node;
529 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT))
530 i_size_write(dn->inode,
531 ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT));
532 return 0;
533 }
534
535 static int __allocate_data_blocks(struct inode *inode, loff_t offset,
536 size_t count)
537 {
538 struct f2fs_map_blocks map;
539
540 map.m_lblk = F2FS_BYTES_TO_BLK(offset);
541 map.m_len = F2FS_BYTES_TO_BLK(count);
542 map.m_next_pgofs = NULL;
543
544 return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_DIO);
545 }
546
547 /*
548 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
549 * f2fs_map_blocks structure.
550 * If original data blocks are allocated, then give them to blockdev.
551 * Otherwise,
552 * a. preallocate requested block addresses
553 * b. do not use extent cache for better performance
554 * c. give the block addresses to blockdev
555 */
556 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
557 int create, int flag)
558 {
559 unsigned int maxblocks = map->m_len;
560 struct dnode_of_data dn;
561 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
562 int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA;
563 pgoff_t pgofs, end_offset;
564 int err = 0, ofs = 1;
565 struct extent_info ei;
566 bool allocated = false;
567 block_t blkaddr;
568
569 map->m_len = 0;
570 map->m_flags = 0;
571
572 /* it only supports block size == page size */
573 pgofs = (pgoff_t)map->m_lblk;
574
575 if (f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
576 map->m_pblk = ei.blk + pgofs - ei.fofs;
577 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
578 map->m_flags = F2FS_MAP_MAPPED;
579 goto out;
580 }
581
582 next_dnode:
583 if (create)
584 f2fs_lock_op(sbi);
585
586 /* When reading holes, we need its node page */
587 set_new_dnode(&dn, inode, NULL, NULL, 0);
588 err = get_dnode_of_data(&dn, pgofs, mode);
589 if (err) {
590 if (err == -ENOENT) {
591 err = 0;
592 if (map->m_next_pgofs)
593 *map->m_next_pgofs =
594 get_next_page_offset(&dn, pgofs);
595 }
596 goto unlock_out;
597 }
598
599 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
600
601 next_block:
602 blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
603
604 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) {
605 if (create) {
606 if (unlikely(f2fs_cp_error(sbi))) {
607 err = -EIO;
608 goto sync_out;
609 }
610 err = __allocate_data_block(&dn);
611 if (err)
612 goto sync_out;
613 allocated = true;
614 map->m_flags = F2FS_MAP_NEW;
615 blkaddr = dn.data_blkaddr;
616 } else {
617 if (flag == F2FS_GET_BLOCK_FIEMAP &&
618 blkaddr == NULL_ADDR) {
619 if (map->m_next_pgofs)
620 *map->m_next_pgofs = pgofs + 1;
621 }
622 if (flag != F2FS_GET_BLOCK_FIEMAP ||
623 blkaddr != NEW_ADDR) {
624 if (flag == F2FS_GET_BLOCK_BMAP)
625 err = -ENOENT;
626 goto sync_out;
627 }
628 }
629 }
630
631 if (map->m_len == 0) {
632 /* preallocated unwritten block should be mapped for fiemap. */
633 if (blkaddr == NEW_ADDR)
634 map->m_flags |= F2FS_MAP_UNWRITTEN;
635 map->m_flags |= F2FS_MAP_MAPPED;
636
637 map->m_pblk = blkaddr;
638 map->m_len = 1;
639 } else if ((map->m_pblk != NEW_ADDR &&
640 blkaddr == (map->m_pblk + ofs)) ||
641 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR)) {
642 ofs++;
643 map->m_len++;
644 } else {
645 goto sync_out;
646 }
647
648 dn.ofs_in_node++;
649 pgofs++;
650
651 if (map->m_len < maxblocks) {
652 if (dn.ofs_in_node < end_offset)
653 goto next_block;
654
655 if (allocated)
656 sync_inode_page(&dn);
657 f2fs_put_dnode(&dn);
658
659 if (create) {
660 f2fs_unlock_op(sbi);
661 f2fs_balance_fs(sbi, allocated);
662 }
663 allocated = false;
664 goto next_dnode;
665 }
666
667 sync_out:
668 if (allocated)
669 sync_inode_page(&dn);
670 f2fs_put_dnode(&dn);
671 unlock_out:
672 if (create) {
673 f2fs_unlock_op(sbi);
674 f2fs_balance_fs(sbi, allocated);
675 }
676 out:
677 trace_f2fs_map_blocks(inode, map, err);
678 return err;
679 }
680
681 static int __get_data_block(struct inode *inode, sector_t iblock,
682 struct buffer_head *bh, int create, int flag,
683 pgoff_t *next_pgofs)
684 {
685 struct f2fs_map_blocks map;
686 int ret;
687
688 map.m_lblk = iblock;
689 map.m_len = bh->b_size >> inode->i_blkbits;
690 map.m_next_pgofs = next_pgofs;
691
692 ret = f2fs_map_blocks(inode, &map, create, flag);
693 if (!ret) {
694 map_bh(bh, inode->i_sb, map.m_pblk);
695 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
696 bh->b_size = map.m_len << inode->i_blkbits;
697 }
698 return ret;
699 }
700
701 static int get_data_block(struct inode *inode, sector_t iblock,
702 struct buffer_head *bh_result, int create, int flag,
703 pgoff_t *next_pgofs)
704 {
705 return __get_data_block(inode, iblock, bh_result, create,
706 flag, next_pgofs);
707 }
708
709 static int get_data_block_dio(struct inode *inode, sector_t iblock,
710 struct buffer_head *bh_result, int create)
711 {
712 return __get_data_block(inode, iblock, bh_result, create,
713 F2FS_GET_BLOCK_DIO, NULL);
714 }
715
716 static int get_data_block_bmap(struct inode *inode, sector_t iblock,
717 struct buffer_head *bh_result, int create)
718 {
719 /* Block number less than F2FS MAX BLOCKS */
720 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
721 return -EFBIG;
722
723 return __get_data_block(inode, iblock, bh_result, create,
724 F2FS_GET_BLOCK_BMAP, NULL);
725 }
726
727 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
728 {
729 return (offset >> inode->i_blkbits);
730 }
731
732 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
733 {
734 return (blk << inode->i_blkbits);
735 }
736
737 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
738 u64 start, u64 len)
739 {
740 struct buffer_head map_bh;
741 sector_t start_blk, last_blk;
742 pgoff_t next_pgofs;
743 loff_t isize;
744 u64 logical = 0, phys = 0, size = 0;
745 u32 flags = 0;
746 int ret = 0;
747
748 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC);
749 if (ret)
750 return ret;
751
752 if (f2fs_has_inline_data(inode)) {
753 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
754 if (ret != -EAGAIN)
755 return ret;
756 }
757
758 inode_lock(inode);
759
760 isize = i_size_read(inode);
761 if (start >= isize)
762 goto out;
763
764 if (start + len > isize)
765 len = isize - start;
766
767 if (logical_to_blk(inode, len) == 0)
768 len = blk_to_logical(inode, 1);
769
770 start_blk = logical_to_blk(inode, start);
771 last_blk = logical_to_blk(inode, start + len - 1);
772
773 next:
774 memset(&map_bh, 0, sizeof(struct buffer_head));
775 map_bh.b_size = len;
776
777 ret = get_data_block(inode, start_blk, &map_bh, 0,
778 F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
779 if (ret)
780 goto out;
781
782 /* HOLE */
783 if (!buffer_mapped(&map_bh)) {
784 start_blk = next_pgofs;
785 /* Go through holes util pass the EOF */
786 if (blk_to_logical(inode, start_blk) < isize)
787 goto prep_next;
788 /* Found a hole beyond isize means no more extents.
789 * Note that the premise is that filesystems don't
790 * punch holes beyond isize and keep size unchanged.
791 */
792 flags |= FIEMAP_EXTENT_LAST;
793 }
794
795 if (size) {
796 if (f2fs_encrypted_inode(inode))
797 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
798
799 ret = fiemap_fill_next_extent(fieinfo, logical,
800 phys, size, flags);
801 }
802
803 if (start_blk > last_blk || ret)
804 goto out;
805
806 logical = blk_to_logical(inode, start_blk);
807 phys = blk_to_logical(inode, map_bh.b_blocknr);
808 size = map_bh.b_size;
809 flags = 0;
810 if (buffer_unwritten(&map_bh))
811 flags = FIEMAP_EXTENT_UNWRITTEN;
812
813 start_blk += logical_to_blk(inode, size);
814
815 prep_next:
816 cond_resched();
817 if (fatal_signal_pending(current))
818 ret = -EINTR;
819 else
820 goto next;
821 out:
822 if (ret == 1)
823 ret = 0;
824
825 inode_unlock(inode);
826 return ret;
827 }
828
829 /*
830 * This function was originally taken from fs/mpage.c, and customized for f2fs.
831 * Major change was from block_size == page_size in f2fs by default.
832 */
833 static int f2fs_mpage_readpages(struct address_space *mapping,
834 struct list_head *pages, struct page *page,
835 unsigned nr_pages)
836 {
837 struct bio *bio = NULL;
838 unsigned page_idx;
839 sector_t last_block_in_bio = 0;
840 struct inode *inode = mapping->host;
841 const unsigned blkbits = inode->i_blkbits;
842 const unsigned blocksize = 1 << blkbits;
843 sector_t block_in_file;
844 sector_t last_block;
845 sector_t last_block_in_file;
846 sector_t block_nr;
847 struct block_device *bdev = inode->i_sb->s_bdev;
848 struct f2fs_map_blocks map;
849
850 map.m_pblk = 0;
851 map.m_lblk = 0;
852 map.m_len = 0;
853 map.m_flags = 0;
854 map.m_next_pgofs = NULL;
855
856 for (page_idx = 0; nr_pages; page_idx++, nr_pages--) {
857
858 prefetchw(&page->flags);
859 if (pages) {
860 page = list_entry(pages->prev, struct page, lru);
861 list_del(&page->lru);
862 if (add_to_page_cache_lru(page, mapping,
863 page->index, GFP_KERNEL))
864 goto next_page;
865 }
866
867 block_in_file = (sector_t)page->index;
868 last_block = block_in_file + nr_pages;
869 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
870 blkbits;
871 if (last_block > last_block_in_file)
872 last_block = last_block_in_file;
873
874 /*
875 * Map blocks using the previous result first.
876 */
877 if ((map.m_flags & F2FS_MAP_MAPPED) &&
878 block_in_file > map.m_lblk &&
879 block_in_file < (map.m_lblk + map.m_len))
880 goto got_it;
881
882 /*
883 * Then do more f2fs_map_blocks() calls until we are
884 * done with this page.
885 */
886 map.m_flags = 0;
887
888 if (block_in_file < last_block) {
889 map.m_lblk = block_in_file;
890 map.m_len = last_block - block_in_file;
891
892 if (f2fs_map_blocks(inode, &map, 0,
893 F2FS_GET_BLOCK_READ))
894 goto set_error_page;
895 }
896 got_it:
897 if ((map.m_flags & F2FS_MAP_MAPPED)) {
898 block_nr = map.m_pblk + block_in_file - map.m_lblk;
899 SetPageMappedToDisk(page);
900
901 if (!PageUptodate(page) && !cleancache_get_page(page)) {
902 SetPageUptodate(page);
903 goto confused;
904 }
905 } else {
906 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
907 SetPageUptodate(page);
908 unlock_page(page);
909 goto next_page;
910 }
911
912 /*
913 * This page will go to BIO. Do we need to send this
914 * BIO off first?
915 */
916 if (bio && (last_block_in_bio != block_nr - 1)) {
917 submit_and_realloc:
918 submit_bio(READ, bio);
919 bio = NULL;
920 }
921 if (bio == NULL) {
922 struct f2fs_crypto_ctx *ctx = NULL;
923
924 if (f2fs_encrypted_inode(inode) &&
925 S_ISREG(inode->i_mode)) {
926
927 ctx = f2fs_get_crypto_ctx(inode);
928 if (IS_ERR(ctx))
929 goto set_error_page;
930
931 /* wait the page to be moved by cleaning */
932 f2fs_wait_on_encrypted_page_writeback(
933 F2FS_I_SB(inode), block_nr);
934 }
935
936 bio = bio_alloc(GFP_KERNEL,
937 min_t(int, nr_pages, BIO_MAX_PAGES));
938 if (!bio) {
939 if (ctx)
940 f2fs_release_crypto_ctx(ctx);
941 goto set_error_page;
942 }
943 bio->bi_bdev = bdev;
944 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(block_nr);
945 bio->bi_end_io = f2fs_read_end_io;
946 bio->bi_private = ctx;
947 }
948
949 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
950 goto submit_and_realloc;
951
952 last_block_in_bio = block_nr;
953 goto next_page;
954 set_error_page:
955 SetPageError(page);
956 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
957 unlock_page(page);
958 goto next_page;
959 confused:
960 if (bio) {
961 submit_bio(READ, bio);
962 bio = NULL;
963 }
964 unlock_page(page);
965 next_page:
966 if (pages)
967 page_cache_release(page);
968 }
969 BUG_ON(pages && !list_empty(pages));
970 if (bio)
971 submit_bio(READ, bio);
972 return 0;
973 }
974
975 static int f2fs_read_data_page(struct file *file, struct page *page)
976 {
977 struct inode *inode = page->mapping->host;
978 int ret = -EAGAIN;
979
980 trace_f2fs_readpage(page, DATA);
981
982 /* If the file has inline data, try to read it directly */
983 if (f2fs_has_inline_data(inode))
984 ret = f2fs_read_inline_data(inode, page);
985 if (ret == -EAGAIN)
986 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
987 return ret;
988 }
989
990 static int f2fs_read_data_pages(struct file *file,
991 struct address_space *mapping,
992 struct list_head *pages, unsigned nr_pages)
993 {
994 struct inode *inode = file->f_mapping->host;
995 struct page *page = list_entry(pages->prev, struct page, lru);
996
997 trace_f2fs_readpages(inode, page, nr_pages);
998
999 /* If the file has inline data, skip readpages */
1000 if (f2fs_has_inline_data(inode))
1001 return 0;
1002
1003 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
1004 }
1005
1006 int do_write_data_page(struct f2fs_io_info *fio)
1007 {
1008 struct page *page = fio->page;
1009 struct inode *inode = page->mapping->host;
1010 struct dnode_of_data dn;
1011 int err = 0;
1012
1013 set_new_dnode(&dn, inode, NULL, NULL, 0);
1014 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1015 if (err)
1016 return err;
1017
1018 fio->blk_addr = dn.data_blkaddr;
1019
1020 /* This page is already truncated */
1021 if (fio->blk_addr == NULL_ADDR) {
1022 ClearPageUptodate(page);
1023 goto out_writepage;
1024 }
1025
1026 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
1027
1028 /* wait for GCed encrypted page writeback */
1029 f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode),
1030 fio->blk_addr);
1031
1032 fio->encrypted_page = f2fs_encrypt(inode, fio->page);
1033 if (IS_ERR(fio->encrypted_page)) {
1034 err = PTR_ERR(fio->encrypted_page);
1035 goto out_writepage;
1036 }
1037 }
1038
1039 set_page_writeback(page);
1040
1041 /*
1042 * If current allocation needs SSR,
1043 * it had better in-place writes for updated data.
1044 */
1045 if (unlikely(fio->blk_addr != NEW_ADDR &&
1046 !is_cold_data(page) &&
1047 !IS_ATOMIC_WRITTEN_PAGE(page) &&
1048 need_inplace_update(inode))) {
1049 rewrite_data_page(fio);
1050 set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE);
1051 trace_f2fs_do_write_data_page(page, IPU);
1052 } else {
1053 write_data_page(&dn, fio);
1054 set_data_blkaddr(&dn);
1055 f2fs_update_extent_cache(&dn);
1056 trace_f2fs_do_write_data_page(page, OPU);
1057 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE);
1058 if (page->index == 0)
1059 set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN);
1060 }
1061 out_writepage:
1062 f2fs_put_dnode(&dn);
1063 return err;
1064 }
1065
1066 static int f2fs_write_data_page(struct page *page,
1067 struct writeback_control *wbc)
1068 {
1069 struct inode *inode = page->mapping->host;
1070 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1071 loff_t i_size = i_size_read(inode);
1072 const pgoff_t end_index = ((unsigned long long) i_size)
1073 >> PAGE_CACHE_SHIFT;
1074 unsigned offset = 0;
1075 bool need_balance_fs = false;
1076 int err = 0;
1077 struct f2fs_io_info fio = {
1078 .sbi = sbi,
1079 .type = DATA,
1080 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
1081 .page = page,
1082 .encrypted_page = NULL,
1083 };
1084
1085 trace_f2fs_writepage(page, DATA);
1086
1087 if (page->index < end_index)
1088 goto write;
1089
1090 /*
1091 * If the offset is out-of-range of file size,
1092 * this page does not have to be written to disk.
1093 */
1094 offset = i_size & (PAGE_CACHE_SIZE - 1);
1095 if ((page->index >= end_index + 1) || !offset)
1096 goto out;
1097
1098 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
1099 write:
1100 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1101 goto redirty_out;
1102 if (f2fs_is_drop_cache(inode))
1103 goto out;
1104 if (f2fs_is_volatile_file(inode) && !wbc->for_reclaim &&
1105 available_free_memory(sbi, BASE_CHECK))
1106 goto redirty_out;
1107
1108 /* Dentry blocks are controlled by checkpoint */
1109 if (S_ISDIR(inode->i_mode)) {
1110 if (unlikely(f2fs_cp_error(sbi)))
1111 goto redirty_out;
1112 err = do_write_data_page(&fio);
1113 goto done;
1114 }
1115
1116 /* we should bypass data pages to proceed the kworkder jobs */
1117 if (unlikely(f2fs_cp_error(sbi))) {
1118 SetPageError(page);
1119 goto out;
1120 }
1121
1122 if (!wbc->for_reclaim)
1123 need_balance_fs = true;
1124 else if (has_not_enough_free_secs(sbi, 0))
1125 goto redirty_out;
1126
1127 err = -EAGAIN;
1128 f2fs_lock_op(sbi);
1129 if (f2fs_has_inline_data(inode))
1130 err = f2fs_write_inline_data(inode, page);
1131 if (err == -EAGAIN)
1132 err = do_write_data_page(&fio);
1133 f2fs_unlock_op(sbi);
1134 done:
1135 if (err && err != -ENOENT)
1136 goto redirty_out;
1137
1138 clear_cold_data(page);
1139 out:
1140 inode_dec_dirty_pages(inode);
1141 if (err)
1142 ClearPageUptodate(page);
1143 unlock_page(page);
1144 f2fs_balance_fs(sbi, need_balance_fs);
1145 if (wbc->for_reclaim || unlikely(f2fs_cp_error(sbi))) {
1146 f2fs_submit_merged_bio(sbi, DATA, WRITE);
1147 remove_dirty_inode(inode);
1148 }
1149 return 0;
1150
1151 redirty_out:
1152 redirty_page_for_writepage(wbc, page);
1153 return AOP_WRITEPAGE_ACTIVATE;
1154 }
1155
1156 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc,
1157 void *data)
1158 {
1159 struct address_space *mapping = data;
1160 int ret = mapping->a_ops->writepage(page, wbc);
1161 mapping_set_error(mapping, ret);
1162 return ret;
1163 }
1164
1165 /*
1166 * This function was copied from write_cche_pages from mm/page-writeback.c.
1167 * The major change is making write step of cold data page separately from
1168 * warm/hot data page.
1169 */
1170 static int f2fs_write_cache_pages(struct address_space *mapping,
1171 struct writeback_control *wbc, writepage_t writepage,
1172 void *data)
1173 {
1174 int ret = 0;
1175 int done = 0;
1176 struct pagevec pvec;
1177 int nr_pages;
1178 pgoff_t uninitialized_var(writeback_index);
1179 pgoff_t index;
1180 pgoff_t end; /* Inclusive */
1181 pgoff_t done_index;
1182 int cycled;
1183 int range_whole = 0;
1184 int tag;
1185 int step = 0;
1186
1187 pagevec_init(&pvec, 0);
1188 next:
1189 if (wbc->range_cyclic) {
1190 writeback_index = mapping->writeback_index; /* prev offset */
1191 index = writeback_index;
1192 if (index == 0)
1193 cycled = 1;
1194 else
1195 cycled = 0;
1196 end = -1;
1197 } else {
1198 index = wbc->range_start >> PAGE_CACHE_SHIFT;
1199 end = wbc->range_end >> PAGE_CACHE_SHIFT;
1200 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1201 range_whole = 1;
1202 cycled = 1; /* ignore range_cyclic tests */
1203 }
1204 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1205 tag = PAGECACHE_TAG_TOWRITE;
1206 else
1207 tag = PAGECACHE_TAG_DIRTY;
1208 retry:
1209 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1210 tag_pages_for_writeback(mapping, index, end);
1211 done_index = index;
1212 while (!done && (index <= end)) {
1213 int i;
1214
1215 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
1216 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1);
1217 if (nr_pages == 0)
1218 break;
1219
1220 for (i = 0; i < nr_pages; i++) {
1221 struct page *page = pvec.pages[i];
1222
1223 if (page->index > end) {
1224 done = 1;
1225 break;
1226 }
1227
1228 done_index = page->index;
1229
1230 lock_page(page);
1231
1232 if (unlikely(page->mapping != mapping)) {
1233 continue_unlock:
1234 unlock_page(page);
1235 continue;
1236 }
1237
1238 if (!PageDirty(page)) {
1239 /* someone wrote it for us */
1240 goto continue_unlock;
1241 }
1242
1243 if (step == is_cold_data(page))
1244 goto continue_unlock;
1245
1246 if (PageWriteback(page)) {
1247 if (wbc->sync_mode != WB_SYNC_NONE)
1248 f2fs_wait_on_page_writeback(page,
1249 DATA, true);
1250 else
1251 goto continue_unlock;
1252 }
1253
1254 BUG_ON(PageWriteback(page));
1255 if (!clear_page_dirty_for_io(page))
1256 goto continue_unlock;
1257
1258 ret = (*writepage)(page, wbc, data);
1259 if (unlikely(ret)) {
1260 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1261 unlock_page(page);
1262 ret = 0;
1263 } else {
1264 done_index = page->index + 1;
1265 done = 1;
1266 break;
1267 }
1268 }
1269
1270 if (--wbc->nr_to_write <= 0 &&
1271 wbc->sync_mode == WB_SYNC_NONE) {
1272 done = 1;
1273 break;
1274 }
1275 }
1276 pagevec_release(&pvec);
1277 cond_resched();
1278 }
1279
1280 if (step < 1) {
1281 step++;
1282 goto next;
1283 }
1284
1285 if (!cycled && !done) {
1286 cycled = 1;
1287 index = 0;
1288 end = writeback_index - 1;
1289 goto retry;
1290 }
1291 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1292 mapping->writeback_index = done_index;
1293
1294 return ret;
1295 }
1296
1297 static int f2fs_write_data_pages(struct address_space *mapping,
1298 struct writeback_control *wbc)
1299 {
1300 struct inode *inode = mapping->host;
1301 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1302 bool locked = false;
1303 int ret;
1304 long diff;
1305
1306 trace_f2fs_writepages(mapping->host, wbc, DATA);
1307
1308 /* deal with chardevs and other special file */
1309 if (!mapping->a_ops->writepage)
1310 return 0;
1311
1312 /* skip writing if there is no dirty page in this inode */
1313 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
1314 return 0;
1315
1316 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
1317 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
1318 available_free_memory(sbi, DIRTY_DENTS))
1319 goto skip_write;
1320
1321 /* skip writing during file defragment */
1322 if (is_inode_flag_set(F2FS_I(inode), FI_DO_DEFRAG))
1323 goto skip_write;
1324
1325 /* during POR, we don't need to trigger writepage at all. */
1326 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1327 goto skip_write;
1328
1329 diff = nr_pages_to_write(sbi, DATA, wbc);
1330
1331 if (!S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_ALL) {
1332 mutex_lock(&sbi->writepages);
1333 locked = true;
1334 }
1335 ret = f2fs_write_cache_pages(mapping, wbc, __f2fs_writepage, mapping);
1336 f2fs_submit_merged_bio(sbi, DATA, WRITE);
1337 if (locked)
1338 mutex_unlock(&sbi->writepages);
1339
1340 remove_dirty_inode(inode);
1341
1342 wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
1343 return ret;
1344
1345 skip_write:
1346 wbc->pages_skipped += get_dirty_pages(inode);
1347 return 0;
1348 }
1349
1350 static void f2fs_write_failed(struct address_space *mapping, loff_t to)
1351 {
1352 struct inode *inode = mapping->host;
1353 loff_t i_size = i_size_read(inode);
1354
1355 if (to > i_size) {
1356 truncate_pagecache(inode, i_size);
1357 truncate_blocks(inode, i_size, true);
1358 }
1359 }
1360
1361 static int prepare_write_begin(struct f2fs_sb_info *sbi,
1362 struct page *page, loff_t pos, unsigned len,
1363 block_t *blk_addr, bool *node_changed)
1364 {
1365 struct inode *inode = page->mapping->host;
1366 pgoff_t index = page->index;
1367 struct dnode_of_data dn;
1368 struct page *ipage;
1369 bool locked = false;
1370 struct extent_info ei;
1371 int err = 0;
1372
1373 if (f2fs_has_inline_data(inode) ||
1374 (pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
1375 f2fs_lock_op(sbi);
1376 locked = true;
1377 }
1378 restart:
1379 /* check inline_data */
1380 ipage = get_node_page(sbi, inode->i_ino);
1381 if (IS_ERR(ipage)) {
1382 err = PTR_ERR(ipage);
1383 goto unlock_out;
1384 }
1385
1386 set_new_dnode(&dn, inode, ipage, ipage, 0);
1387
1388 if (f2fs_has_inline_data(inode)) {
1389 if (pos + len <= MAX_INLINE_DATA) {
1390 read_inline_data(page, ipage);
1391 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
1392 set_inline_node(ipage);
1393 } else {
1394 err = f2fs_convert_inline_page(&dn, page);
1395 if (err)
1396 goto out;
1397 if (dn.data_blkaddr == NULL_ADDR)
1398 err = f2fs_get_block(&dn, index);
1399 }
1400 } else if (locked) {
1401 err = f2fs_get_block(&dn, index);
1402 } else {
1403 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1404 dn.data_blkaddr = ei.blk + index - ei.fofs;
1405 } else {
1406 /* hole case */
1407 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
1408 if (err || (!err && dn.data_blkaddr == NULL_ADDR)) {
1409 f2fs_put_dnode(&dn);
1410 f2fs_lock_op(sbi);
1411 locked = true;
1412 goto restart;
1413 }
1414 }
1415 }
1416
1417 /* convert_inline_page can make node_changed */
1418 *blk_addr = dn.data_blkaddr;
1419 *node_changed = dn.node_changed;
1420 out:
1421 f2fs_put_dnode(&dn);
1422 unlock_out:
1423 if (locked)
1424 f2fs_unlock_op(sbi);
1425 return err;
1426 }
1427
1428 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
1429 loff_t pos, unsigned len, unsigned flags,
1430 struct page **pagep, void **fsdata)
1431 {
1432 struct inode *inode = mapping->host;
1433 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1434 struct page *page = NULL;
1435 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
1436 bool need_balance = false;
1437 block_t blkaddr = NULL_ADDR;
1438 int err = 0;
1439
1440 trace_f2fs_write_begin(inode, pos, len, flags);
1441
1442 /*
1443 * We should check this at this moment to avoid deadlock on inode page
1444 * and #0 page. The locking rule for inline_data conversion should be:
1445 * lock_page(page #0) -> lock_page(inode_page)
1446 */
1447 if (index != 0) {
1448 err = f2fs_convert_inline_inode(inode);
1449 if (err)
1450 goto fail;
1451 }
1452 repeat:
1453 page = grab_cache_page_write_begin(mapping, index, flags);
1454 if (!page) {
1455 err = -ENOMEM;
1456 goto fail;
1457 }
1458
1459 *pagep = page;
1460
1461 err = prepare_write_begin(sbi, page, pos, len,
1462 &blkaddr, &need_balance);
1463 if (err)
1464 goto fail;
1465
1466 if (need_balance && has_not_enough_free_secs(sbi, 0)) {
1467 unlock_page(page);
1468 f2fs_balance_fs(sbi, true);
1469 lock_page(page);
1470 if (page->mapping != mapping) {
1471 /* The page got truncated from under us */
1472 f2fs_put_page(page, 1);
1473 goto repeat;
1474 }
1475 }
1476
1477 f2fs_wait_on_page_writeback(page, DATA, false);
1478
1479 /* wait for GCed encrypted page writeback */
1480 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
1481 f2fs_wait_on_encrypted_page_writeback(sbi, blkaddr);
1482
1483 if (len == PAGE_CACHE_SIZE)
1484 goto out_update;
1485 if (PageUptodate(page))
1486 goto out_clear;
1487
1488 if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
1489 unsigned start = pos & (PAGE_CACHE_SIZE - 1);
1490 unsigned end = start + len;
1491
1492 /* Reading beyond i_size is simple: memset to zero */
1493 zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE);
1494 goto out_update;
1495 }
1496
1497 if (blkaddr == NEW_ADDR) {
1498 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
1499 } else {
1500 struct f2fs_io_info fio = {
1501 .sbi = sbi,
1502 .type = DATA,
1503 .rw = READ_SYNC,
1504 .blk_addr = blkaddr,
1505 .page = page,
1506 .encrypted_page = NULL,
1507 };
1508 err = f2fs_submit_page_bio(&fio);
1509 if (err)
1510 goto fail;
1511
1512 lock_page(page);
1513 if (unlikely(!PageUptodate(page))) {
1514 err = -EIO;
1515 goto fail;
1516 }
1517 if (unlikely(page->mapping != mapping)) {
1518 f2fs_put_page(page, 1);
1519 goto repeat;
1520 }
1521
1522 /* avoid symlink page */
1523 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
1524 err = f2fs_decrypt_one(inode, page);
1525 if (err)
1526 goto fail;
1527 }
1528 }
1529 out_update:
1530 SetPageUptodate(page);
1531 out_clear:
1532 clear_cold_data(page);
1533 return 0;
1534
1535 fail:
1536 f2fs_put_page(page, 1);
1537 f2fs_write_failed(mapping, pos + len);
1538 return err;
1539 }
1540
1541 static int f2fs_write_end(struct file *file,
1542 struct address_space *mapping,
1543 loff_t pos, unsigned len, unsigned copied,
1544 struct page *page, void *fsdata)
1545 {
1546 struct inode *inode = page->mapping->host;
1547
1548 trace_f2fs_write_end(inode, pos, len, copied);
1549
1550 set_page_dirty(page);
1551
1552 if (pos + copied > i_size_read(inode)) {
1553 i_size_write(inode, pos + copied);
1554 mark_inode_dirty(inode);
1555 }
1556
1557 f2fs_put_page(page, 1);
1558 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
1559 return copied;
1560 }
1561
1562 static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
1563 loff_t offset)
1564 {
1565 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
1566
1567 if (offset & blocksize_mask)
1568 return -EINVAL;
1569
1570 if (iov_iter_alignment(iter) & blocksize_mask)
1571 return -EINVAL;
1572
1573 return 0;
1574 }
1575
1576 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
1577 loff_t offset)
1578 {
1579 struct file *file = iocb->ki_filp;
1580 struct address_space *mapping = file->f_mapping;
1581 struct inode *inode = mapping->host;
1582 size_t count = iov_iter_count(iter);
1583 int err;
1584
1585 /* we don't need to use inline_data strictly */
1586 err = f2fs_convert_inline_inode(inode);
1587 if (err)
1588 return err;
1589
1590 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
1591 return 0;
1592
1593 err = check_direct_IO(inode, iter, offset);
1594 if (err)
1595 return err;
1596
1597 trace_f2fs_direct_IO_enter(inode, offset, count, iov_iter_rw(iter));
1598
1599 if (iov_iter_rw(iter) == WRITE) {
1600 err = __allocate_data_blocks(inode, offset, count);
1601 if (err)
1602 goto out;
1603 }
1604
1605 err = blockdev_direct_IO(iocb, inode, iter, offset, get_data_block_dio);
1606 out:
1607 if (err < 0 && iov_iter_rw(iter) == WRITE)
1608 f2fs_write_failed(mapping, offset + count);
1609
1610 trace_f2fs_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), err);
1611
1612 return err;
1613 }
1614
1615 void f2fs_invalidate_page(struct page *page, unsigned int offset,
1616 unsigned int length)
1617 {
1618 struct inode *inode = page->mapping->host;
1619 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1620
1621 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
1622 (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE))
1623 return;
1624
1625 if (PageDirty(page)) {
1626 if (inode->i_ino == F2FS_META_INO(sbi))
1627 dec_page_count(sbi, F2FS_DIRTY_META);
1628 else if (inode->i_ino == F2FS_NODE_INO(sbi))
1629 dec_page_count(sbi, F2FS_DIRTY_NODES);
1630 else
1631 inode_dec_dirty_pages(inode);
1632 }
1633
1634 /* This is atomic written page, keep Private */
1635 if (IS_ATOMIC_WRITTEN_PAGE(page))
1636 return;
1637
1638 ClearPagePrivate(page);
1639 }
1640
1641 int f2fs_release_page(struct page *page, gfp_t wait)
1642 {
1643 /* If this is dirty page, keep PagePrivate */
1644 if (PageDirty(page))
1645 return 0;
1646
1647 /* This is atomic written page, keep Private */
1648 if (IS_ATOMIC_WRITTEN_PAGE(page))
1649 return 0;
1650
1651 ClearPagePrivate(page);
1652 return 1;
1653 }
1654
1655 static int f2fs_set_data_page_dirty(struct page *page)
1656 {
1657 struct address_space *mapping = page->mapping;
1658 struct inode *inode = mapping->host;
1659
1660 trace_f2fs_set_page_dirty(page, DATA);
1661
1662 SetPageUptodate(page);
1663
1664 if (f2fs_is_atomic_file(inode)) {
1665 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
1666 register_inmem_page(inode, page);
1667 return 1;
1668 }
1669 /*
1670 * Previously, this page has been registered, we just
1671 * return here.
1672 */
1673 return 0;
1674 }
1675
1676 if (!PageDirty(page)) {
1677 __set_page_dirty_nobuffers(page);
1678 update_dirty_page(inode, page);
1679 return 1;
1680 }
1681 return 0;
1682 }
1683
1684 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
1685 {
1686 struct inode *inode = mapping->host;
1687
1688 if (f2fs_has_inline_data(inode))
1689 return 0;
1690
1691 /* make sure allocating whole blocks */
1692 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
1693 filemap_write_and_wait(mapping);
1694
1695 return generic_block_bmap(mapping, block, get_data_block_bmap);
1696 }
1697
1698 const struct address_space_operations f2fs_dblock_aops = {
1699 .readpage = f2fs_read_data_page,
1700 .readpages = f2fs_read_data_pages,
1701 .writepage = f2fs_write_data_page,
1702 .writepages = f2fs_write_data_pages,
1703 .write_begin = f2fs_write_begin,
1704 .write_end = f2fs_write_end,
1705 .set_page_dirty = f2fs_set_data_page_dirty,
1706 .invalidatepage = f2fs_invalidate_page,
1707 .releasepage = f2fs_release_page,
1708 .direct_IO = f2fs_direct_IO,
1709 .bmap = f2fs_bmap,
1710 };
Linux kernel - Deliverables
This page took 0.177724 seconds and 6 git commands to generate.