projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
f2fs crypto: avoid unneeded memory allocation when {en/de}crypting symlink
[deliverable/linux.git] / fs / f2fs / segment.c
CommitLineData
0a8165d7 1/*
351df4b2
JK		 2 * fs/f2fs/segment.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/bio.h>
14#include <linux/blkdev.h>
690e4a3e 15#include <linux/prefetch.h>
6b4afdd7 16#include <linux/kthread.h>
74de593a 17#include <linux/swap.h>
60b99b48 18#include <linux/timer.h>
351df4b2
JK		 19
20#include "f2fs.h"
21#include "segment.h"
22#include "node.h"
9e4ded3f 23#include "trace.h"
6ec178da 24#include <trace/events/f2fs.h>
351df4b2 25
9a7f143a
CL		 26#define __reverse_ffz(x) __reverse_ffs(~(x))
27
7fd9e544 28static struct kmem_cache *discard_entry_slab;
184a5cd2 29static struct kmem_cache *sit_entry_set_slab;
88b88a66 30static struct kmem_cache *inmem_entry_slab;
7fd9e544 31
f96999c3
JK		 32static unsigned long __reverse_ulong(unsigned char *str)
33{
34 unsigned long tmp = 0;
35 int shift = 24, idx = 0;
36
37#if BITS_PER_LONG == 64
38 shift = 56;
39#endif
40 while (shift >= 0) {
41 tmp |= (unsigned long)str[idx++] << shift;
42 shift -= BITS_PER_BYTE;
43 }
44 return tmp;
45}
46
9a7f143a
CL		 47/*
48 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
49 * MSB and LSB are reversed in a byte by f2fs_set_bit.
50 */
51static inline unsigned long __reverse_ffs(unsigned long word)
52{
53 int num = 0;
54
55#if BITS_PER_LONG == 64
f96999c3 56 if ((word & 0xffffffff00000000UL) == 0)
9a7f143a 57 num += 32;
f96999c3 58 else
9a7f143a 59 word >>= 32;
9a7f143a 60#endif
f96999c3 61 if ((word & 0xffff0000) == 0)
9a7f143a 62 num += 16;
f96999c3 63 else
9a7f143a 64 word >>= 16;
f96999c3
JK		 65
66 if ((word & 0xff00) == 0)
9a7f143a 67 num += 8;
f96999c3 68 else
9a7f143a 69 word >>= 8;
f96999c3 70
9a7f143a
CL		 71 if ((word & 0xf0) == 0)
72 num += 4;
73 else
74 word >>= 4;
f96999c3 75
9a7f143a
CL		 76 if ((word & 0xc) == 0)
77 num += 2;
78 else
79 word >>= 2;
f96999c3 80
9a7f143a
CL		 81 if ((word & 0x2) == 0)
82 num += 1;
83 return num;
84}
85
86/*
e1c42045 87 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
9a7f143a 88 * f2fs_set_bit makes MSB and LSB reversed in a byte.
692223d1 89 * @size must be integral times of unsigned long.
9a7f143a 90 * Example:
f96999c3
JK		 91 * MSB <--> LSB
92 * f2fs_set_bit(0, bitmap) => 1000 0000
93 * f2fs_set_bit(7, bitmap) => 0000 0001
9a7f143a
CL		 94 */
95static unsigned long __find_rev_next_bit(const unsigned long *addr,
96 unsigned long size, unsigned long offset)
97{
98 const unsigned long *p = addr + BIT_WORD(offset);
692223d1 99 unsigned long result = size;
9a7f143a 100 unsigned long tmp;
9a7f143a
CL		 101
102 if (offset >= size)
103 return size;
104
692223d1 105 size -= (offset & ~(BITS_PER_LONG - 1));
9a7f143a 106 offset %= BITS_PER_LONG;
f96999c3 107
692223d1
FL		 108 while (1) {
109 if (*p == 0)
110 goto pass;
9a7f143a 111
f96999c3 112 tmp = __reverse_ulong((unsigned char *)p);
692223d1
FL		 113
114 tmp &= ~0UL >> offset;
115 if (size < BITS_PER_LONG)
116 tmp &= (~0UL << (BITS_PER_LONG - size));
9a7f143a 117 if (tmp)
692223d1
FL		 118 goto found;
119pass:
120 if (size <= BITS_PER_LONG)
121 break;
9a7f143a 122 size -= BITS_PER_LONG;
692223d1 123 offset = 0;
f96999c3 124 p++;
9a7f143a 125 }
692223d1
FL		 126 return result;
127found:
128 return result - size + __reverse_ffs(tmp);
9a7f143a
CL		 129}
130
131static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
132 unsigned long size, unsigned long offset)
133{
134 const unsigned long *p = addr + BIT_WORD(offset);
80609448 135 unsigned long result = size;
9a7f143a 136 unsigned long tmp;
9a7f143a
CL		 137
138 if (offset >= size)
139 return size;
140
80609448 141 size -= (offset & ~(BITS_PER_LONG - 1));
9a7f143a 142 offset %= BITS_PER_LONG;
80609448
JK		 143
144 while (1) {
145 if (*p == ~0UL)
146 goto pass;
147
f96999c3 148 tmp = __reverse_ulong((unsigned char *)p);
80609448
JK		 149
150 if (offset)
151 tmp |= ~0UL << (BITS_PER_LONG - offset);
152 if (size < BITS_PER_LONG)
153 tmp |= ~0UL >> size;
f96999c3 154 if (tmp != ~0UL)
80609448
JK		 155 goto found;
156pass:
157 if (size <= BITS_PER_LONG)
158 break;
9a7f143a 159 size -= BITS_PER_LONG;
80609448 160 offset = 0;
f96999c3 161 p++;
9a7f143a 162 }
80609448
JK		 163 return result;
164found:
165 return result - size + __reverse_ffz(tmp);
9a7f143a
CL		 166}
167
88b88a66
JK		 168void register_inmem_page(struct inode *inode, struct page *page)
169{
170 struct f2fs_inode_info *fi = F2FS_I(inode);
171 struct inmem_pages *new;
9be32d72 172
9e4ded3f 173 f2fs_trace_pid(page);
0722b101 174
decd36b6
CY		 175 set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
176 SetPagePrivate(page);
177
88b88a66
JK		 178 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
179
180 /* add atomic page indices to the list */
181 new->page = page;
182 INIT_LIST_HEAD(&new->list);
decd36b6 183
88b88a66
JK		 184 /* increase reference count with clean state */
185 mutex_lock(&fi->inmem_lock);
186 get_page(page);
187 list_add_tail(&new->list, &fi->inmem_pages);
8dcf2ff7 188 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
88b88a66 189 mutex_unlock(&fi->inmem_lock);
8ce67cb0
JK		 190
191 trace_f2fs_register_inmem_page(page, INMEM);
88b88a66
JK		 192}
193
28bc106b
CY		 194static int __revoke_inmem_pages(struct inode *inode,
195 struct list_head *head, bool drop, bool recover)
29b96b54 196{
28bc106b 197 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
29b96b54 198 struct inmem_pages *cur, *tmp;
28bc106b 199 int err = 0;
29b96b54
CY		 200
201 list_for_each_entry_safe(cur, tmp, head, list) {
28bc106b
CY		 202 struct page *page = cur->page;
203
204 if (drop)
205 trace_f2fs_commit_inmem_page(page, INMEM_DROP);
206
207 lock_page(page);
29b96b54 208
28bc106b
CY		 209 if (recover) {
210 struct dnode_of_data dn;
211 struct node_info ni;
212
213 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
214
215 set_new_dnode(&dn, inode, NULL, NULL, 0);
216 if (get_dnode_of_data(&dn, page->index, LOOKUP_NODE)) {
217 err = -EAGAIN;
218 goto next;
219 }
220 get_node_info(sbi, dn.nid, &ni);
221 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
222 cur->old_addr, ni.version, true, true);
223 f2fs_put_dnode(&dn);
224 }
225next:
226 ClearPageUptodate(page);
227 set_page_private(page, 0);
228 ClearPageUptodate(page);
229 f2fs_put_page(page, 1);
29b96b54
CY		 230
231 list_del(&cur->list);
232 kmem_cache_free(inmem_entry_slab, cur);
233 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
234 }
28bc106b 235 return err;
29b96b54
CY		 236}
237
238void drop_inmem_pages(struct inode *inode)
239{
240 struct f2fs_inode_info *fi = F2FS_I(inode);
241
242 mutex_lock(&fi->inmem_lock);
28bc106b 243 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
29b96b54
CY		 244 mutex_unlock(&fi->inmem_lock);
245}
246
28bc106b
CY		 247static int __commit_inmem_pages(struct inode *inode,
248 struct list_head *revoke_list)
88b88a66
JK		 249{
250 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
251 struct f2fs_inode_info *fi = F2FS_I(inode);
252 struct inmem_pages *cur, *tmp;
88b88a66 253 struct f2fs_io_info fio = {
05ca3632 254 .sbi = sbi,
88b88a66 255 .type = DATA,
1e84371f 256 .rw = WRITE_SYNC | REQ_PRIO,
4375a336 257 .encrypted_page = NULL,
88b88a66 258 };
29b96b54 259 bool submit_bio = false;
edb27dee 260 int err = 0;
88b88a66 261
88b88a66 262 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
28bc106b
CY		 263 struct page *page = cur->page;
264
265 lock_page(page);
266 if (page->mapping == inode->i_mapping) {
267 trace_f2fs_commit_inmem_page(page, INMEM);
268
269 set_page_dirty(page);
270 f2fs_wait_on_page_writeback(page, DATA, true);
271 if (clear_page_dirty_for_io(page))
29b96b54 272 inode_dec_dirty_pages(inode);
28bc106b
CY		 273
274 fio.page = page;
29b96b54
CY		 275 err = do_write_data_page(&fio);
276 if (err) {
28bc106b 277 unlock_page(page);
29b96b54 278 break;
70c640b1 279 }
29b96b54 280
28bc106b
CY		 281 /* record old blkaddr for revoking */
282 cur->old_addr = fio.old_blkaddr;
decd36b6 283
28bc106b
CY		 284 clear_cold_data(page);
285 submit_bio = true;
286 }
287 unlock_page(page);
288 list_move_tail(&cur->list, revoke_list);
88b88a66 289 }
29b96b54
CY		 290
291 if (submit_bio)
292 f2fs_submit_merged_bio_cond(sbi, inode, NULL, 0, DATA, WRITE);
28bc106b
CY		 293
294 if (!err)
295 __revoke_inmem_pages(inode, revoke_list, false, false);
296
29b96b54
CY		 297 return err;
298}
299
300int commit_inmem_pages(struct inode *inode)
301{
302 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
303 struct f2fs_inode_info *fi = F2FS_I(inode);
28bc106b
CY		 304 struct list_head revoke_list;
305 int err;
29b96b54 306
28bc106b 307 INIT_LIST_HEAD(&revoke_list);
29b96b54
CY		 308 f2fs_balance_fs(sbi, true);
309 f2fs_lock_op(sbi);
310
311 mutex_lock(&fi->inmem_lock);
28bc106b
CY		 312 err = __commit_inmem_pages(inode, &revoke_list);
313 if (err) {
314 int ret;
315 /*
316 * try to revoke all committed pages, but still we could fail
317 * due to no memory or other reason, if that happened, EAGAIN
318 * will be returned, which means in such case, transaction is
319 * already not integrity, caller should use journal to do the
320 * recovery or rewrite & commit last transaction. For other
321 * error number, revoking was done by filesystem itself.
322 */
323 ret = __revoke_inmem_pages(inode, &revoke_list, false, true);
324 if (ret)
325 err = ret;
326
327 /* drop all uncommitted pages */
328 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
329 }
88b88a66
JK		 330 mutex_unlock(&fi->inmem_lock);
331
29b96b54 332 f2fs_unlock_op(sbi);
edb27dee 333 return err;
88b88a66
JK		 334}
335
0a8165d7 336/*
351df4b2
JK		 337 * This function balances dirty node and dentry pages.
338 * In addition, it controls garbage collection.
339 */
2c4db1a6 340void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
351df4b2 341{
2c4db1a6
JK		 342 if (!need)
343 return;
351df4b2 344 /*
029cd28c
JK		 345 * We should do GC or end up with checkpoint, if there are so many dirty
346 * dir/node pages without enough free segments.
351df4b2 347 */
43727527 348 if (has_not_enough_free_secs(sbi, 0)) {
351df4b2 349 mutex_lock(&sbi->gc_mutex);
d530d4d8 350 f2fs_gc(sbi, false);
351df4b2
JK		 351 }
352}
353
4660f9c0
JK		 354void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
355{
1dcc336b 356 /* try to shrink extent cache when there is no enough memory */
554df79e
JK		 357 if (!available_free_memory(sbi, EXTENT_CACHE))
358 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
1dcc336b 359
1b38dc8e
JK		 360 /* check the # of cached NAT entries */
361 if (!available_free_memory(sbi, NAT_ENTRIES))
362 try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
363
31696580
CY		 364 if (!available_free_memory(sbi, FREE_NIDS))
365 try_to_free_nids(sbi, NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES);
366
1b38dc8e
JK		 367 /* checkpoint is the only way to shrink partial cached entries */
368 if (!available_free_memory(sbi, NAT_ENTRIES) ||
60b99b48 369 !available_free_memory(sbi, INO_ENTRIES) ||
7d768d2c
CY		 370 excess_prefree_segs(sbi) ||
371 excess_dirty_nats(sbi) ||
d0239e1b 372 (is_idle(sbi) && f2fs_time_over(sbi, CP_TIME))) {
36b35a0d
CY		 373 if (test_opt(sbi, DATA_FLUSH))
374 sync_dirty_inodes(sbi, FILE_INODE);
4660f9c0 375 f2fs_sync_fs(sbi->sb, true);
42190d2a 376 stat_inc_bg_cp_count(sbi->stat_info);
36b35a0d 377 }
4660f9c0
JK		 378}
379
2163d198 380static int issue_flush_thread(void *data)
6b4afdd7
JK		 381{
382 struct f2fs_sb_info *sbi = data;
a688b9d9
GZ		 383 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
384 wait_queue_head_t *q = &fcc->flush_wait_queue;
6b4afdd7
JK		 385repeat:
386 if (kthread_should_stop())
387 return 0;
388
721bd4d5 389 if (!llist_empty(&fcc->issue_list)) {
740432f8 390 struct bio *bio;
6b4afdd7
JK		 391 struct flush_cmd *cmd, *next;
392 int ret;
393
740432f8
JK		 394 bio = f2fs_bio_alloc(0);
395
721bd4d5
GZ		 396 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
397 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
398
6b4afdd7
JK		 399 bio->bi_bdev = sbi->sb->s_bdev;
400 ret = submit_bio_wait(WRITE_FLUSH, bio);
401
721bd4d5
GZ		 402 llist_for_each_entry_safe(cmd, next,
403 fcc->dispatch_list, llnode) {
6b4afdd7 404 cmd->ret = ret;
6b4afdd7
JK		 405 complete(&cmd->wait);
406 }
a4ed23f2 407 bio_put(bio);
a688b9d9 408 fcc->dispatch_list = NULL;
6b4afdd7
JK		 409 }
410
a688b9d9 411 wait_event_interruptible(*q,
721bd4d5 412 kthread_should_stop() || !llist_empty(&fcc->issue_list));
6b4afdd7
JK		 413 goto repeat;
414}
415
416int f2fs_issue_flush(struct f2fs_sb_info *sbi)
417{
a688b9d9 418 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
adf8d90b 419 struct flush_cmd cmd;
6b4afdd7 420
24a9ee0f
JK		 421 trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER),
422 test_opt(sbi, FLUSH_MERGE));
423
0f7b2abd
JK		 424 if (test_opt(sbi, NOBARRIER))
425 return 0;
426
740432f8
JK		 427 if (!test_opt(sbi, FLUSH_MERGE)) {
428 struct bio *bio = f2fs_bio_alloc(0);
429 int ret;
430
431 bio->bi_bdev = sbi->sb->s_bdev;
432 ret = submit_bio_wait(WRITE_FLUSH, bio);
433 bio_put(bio);
434 return ret;
435 }
6b4afdd7 436
adf8d90b 437 init_completion(&cmd.wait);
6b4afdd7 438
721bd4d5 439 llist_add(&cmd.llnode, &fcc->issue_list);
6b4afdd7 440
a688b9d9
GZ		 441 if (!fcc->dispatch_list)
442 wake_up(&fcc->flush_wait_queue);
6b4afdd7 443
adf8d90b
CY		 444 wait_for_completion(&cmd.wait);
445
446 return cmd.ret;
6b4afdd7
JK		 447}
448
2163d198
GZ		 449int create_flush_cmd_control(struct f2fs_sb_info *sbi)
450{
451 dev_t dev = sbi->sb->s_bdev->bd_dev;
452 struct flush_cmd_control *fcc;
453 int err = 0;
454
455 fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL);
456 if (!fcc)
457 return -ENOMEM;
2163d198 458 init_waitqueue_head(&fcc->flush_wait_queue);
721bd4d5 459 init_llist_head(&fcc->issue_list);
6b2920a5 460 SM_I(sbi)->cmd_control_info = fcc;
2163d198
GZ		 461 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
462 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
463 if (IS_ERR(fcc->f2fs_issue_flush)) {
464 err = PTR_ERR(fcc->f2fs_issue_flush);
465 kfree(fcc);
6b2920a5 466 SM_I(sbi)->cmd_control_info = NULL;
2163d198
GZ		 467 return err;
468 }
2163d198
GZ		 469
470 return err;
471}
472
473void destroy_flush_cmd_control(struct f2fs_sb_info *sbi)
474{
6b2920a5 475 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
2163d198
GZ		 476
477 if (fcc && fcc->f2fs_issue_flush)
478 kthread_stop(fcc->f2fs_issue_flush);
479 kfree(fcc);
6b2920a5 480 SM_I(sbi)->cmd_control_info = NULL;
2163d198
GZ		 481}
482
351df4b2
JK		 483static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
484 enum dirty_type dirty_type)
485{
486 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
487
488 /* need not be added */
489 if (IS_CURSEG(sbi, segno))
490 return;
491
492 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
493 dirty_i->nr_dirty[dirty_type]++;
494
495 if (dirty_type == DIRTY) {
496 struct seg_entry *sentry = get_seg_entry(sbi, segno);
4625d6aa 497 enum dirty_type t = sentry->type;
b2f2c390 498
ec325b52
JK		 499 if (unlikely(t >= DIRTY)) {
500 f2fs_bug_on(sbi, 1);
501 return;
502 }
4625d6aa
CL		 503 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
504 dirty_i->nr_dirty[t]++;
351df4b2
JK		 505 }
506}
507
508static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
509 enum dirty_type dirty_type)
510{
511 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
512
513 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
514 dirty_i->nr_dirty[dirty_type]--;
515
516 if (dirty_type == DIRTY) {
4625d6aa
CL		 517 struct seg_entry *sentry = get_seg_entry(sbi, segno);
518 enum dirty_type t = sentry->type;
519
520 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
521 dirty_i->nr_dirty[t]--;
b2f2c390 522
5ec4e49f
JK		 523 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
524 clear_bit(GET_SECNO(sbi, segno),
525 dirty_i->victim_secmap);
351df4b2
JK		 526 }
527}
528
0a8165d7 529/*
351df4b2
JK		 530 * Should not occur error such as -ENOMEM.
531 * Adding dirty entry into seglist is not critical operation.
532 * If a given segment is one of current working segments, it won't be added.
533 */
8d8451af 534static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
351df4b2
JK		 535{
536 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
537 unsigned short valid_blocks;
538
539 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
540 return;
541
542 mutex_lock(&dirty_i->seglist_lock);
543
544 valid_blocks = get_valid_blocks(sbi, segno, 0);
545
546 if (valid_blocks == 0) {
547 __locate_dirty_segment(sbi, segno, PRE);
548 __remove_dirty_segment(sbi, segno, DIRTY);
549 } else if (valid_blocks < sbi->blocks_per_seg) {
550 __locate_dirty_segment(sbi, segno, DIRTY);
551 } else {
552 /* Recovery routine with SSR needs this */
553 __remove_dirty_segment(sbi, segno, DIRTY);
554 }
555
556 mutex_unlock(&dirty_i->seglist_lock);
351df4b2
JK		 557}
558
1e87a78d 559static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
37208879
JK		 560 block_t blkstart, block_t blklen)
561{
55cf9cb6
CY		 562 sector_t start = SECTOR_FROM_BLOCK(blkstart);
563 sector_t len = SECTOR_FROM_BLOCK(blklen);
a66cdd98
JK		 564 struct seg_entry *se;
565 unsigned int offset;
566 block_t i;
567
568 for (i = blkstart; i < blkstart + blklen; i++) {
569 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
570 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
571
572 if (!f2fs_test_and_set_bit(offset, se->discard_map))
573 sbi->discard_blks--;
574 }
1661d07c 575 trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
1e87a78d
JK		 576 return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0);
577}
578
e90c2d28 579bool discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr)
1e87a78d 580{
60b286c4 581 int err = -EOPNOTSUPP;
40a02be1
JK		 582
583 if (test_opt(sbi, DISCARD)) {
584 struct seg_entry *se = get_seg_entry(sbi,
585 GET_SEGNO(sbi, blkaddr));
586 unsigned int offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
587
588 if (f2fs_test_bit(offset, se->discard_map))
e90c2d28 589 return false;
40a02be1
JK		 590
591 err = f2fs_issue_discard(sbi, blkaddr, 1);
592 }
593
e90c2d28 594 if (err) {
381722d2 595 update_meta_page(sbi, NULL, blkaddr);
e90c2d28
CY		 596 return true;
597 }
598 return false;
37208879
JK		 599}
600
adf4983b 601static void __add_discard_entry(struct f2fs_sb_info *sbi,
a66cdd98
JK		 602 struct cp_control *cpc, struct seg_entry *se,
603 unsigned int start, unsigned int end)
b2955550
JK		 604{
605 struct list_head *head = &SM_I(sbi)->discard_list;
adf4983b
JK		 606 struct discard_entry *new, *last;
607
608 if (!list_empty(head)) {
609 last = list_last_entry(head, struct discard_entry, list);
610 if (START_BLOCK(sbi, cpc->trim_start) + start ==
611 last->blkaddr + last->len) {
612 last->len += end - start;
613 goto done;
614 }
615 }
616
617 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
618 INIT_LIST_HEAD(&new->list);
619 new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start;
620 new->len = end - start;
621 list_add_tail(&new->list, head);
622done:
623 SM_I(sbi)->nr_discards += end - start;
adf4983b
JK		 624}
625
626static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
627{
b2955550
JK		 628 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
629 int max_blocks = sbi->blocks_per_seg;
4b2fecc8 630 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
b2955550
JK		 631 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
632 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
a66cdd98 633 unsigned long *discard_map = (unsigned long *)se->discard_map;
60a3b782 634 unsigned long *dmap = SIT_I(sbi)->tmp_map;
b2955550 635 unsigned int start = 0, end = -1;
4b2fecc8 636 bool force = (cpc->reason == CP_DISCARD);
b2955550
JK		 637 int i;
638
a66cdd98 639 if (se->valid_blocks == max_blocks)
b2955550
JK		 640 return;
641
a66cdd98
JK		 642 if (!force) {
643 if (!test_opt(sbi, DISCARD) || !se->valid_blocks ||
912a83b5
DC		 644 SM_I(sbi)->nr_discards >= SM_I(sbi)->max_discards)
645 return;
4b2fecc8
JK		 646 }
647
b2955550
JK		 648 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
649 for (i = 0; i < entries; i++)
a66cdd98 650 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
d7bc2484 651 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
b2955550 652
4b2fecc8 653 while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
b2955550
JK		 654 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
655 if (start >= max_blocks)
656 break;
657
658 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
a66cdd98 659 __add_discard_entry(sbi, cpc, se, start, end);
b2955550
JK		 660 }
661}
662
4b2fecc8
JK		 663void release_discard_addrs(struct f2fs_sb_info *sbi)
664{
665 struct list_head *head = &(SM_I(sbi)->discard_list);
666 struct discard_entry *entry, *this;
667
668 /* drop caches */
669 list_for_each_entry_safe(entry, this, head, list) {
670 list_del(&entry->list);
671 kmem_cache_free(discard_entry_slab, entry);
672 }
673}
674
0a8165d7 675/*
351df4b2
JK		 676 * Should call clear_prefree_segments after checkpoint is done.
677 */
678static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
679{
680 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
b65ee148 681 unsigned int segno;
351df4b2
JK		 682
683 mutex_lock(&dirty_i->seglist_lock);
7cd8558b 684 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
351df4b2 685 __set_test_and_free(sbi, segno);
351df4b2
JK		 686 mutex_unlock(&dirty_i->seglist_lock);
687}
688
836b5a63 689void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)
351df4b2 690{
b2955550 691 struct list_head *head = &(SM_I(sbi)->discard_list);
2d7b822a 692 struct discard_entry *entry, *this;
351df4b2 693 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
29e59c14 694 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
29e59c14 695 unsigned int start = 0, end = -1;
351df4b2
JK		 696
697 mutex_lock(&dirty_i->seglist_lock);
29e59c14 698
351df4b2 699 while (1) {
29e59c14 700 int i;
7cd8558b
JK		 701 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
702 if (start >= MAIN_SEGS(sbi))
351df4b2 703 break;
7cd8558b
JK		 704 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
705 start + 1);
29e59c14
CL		 706
707 for (i = start; i < end; i++)
708 clear_bit(i, prefree_map);
709
710 dirty_i->nr_dirty[PRE] -= end - start;
711
712 if (!test_opt(sbi, DISCARD))
713 continue;
351df4b2 714
37208879
JK		 715 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
716 (end - start) << sbi->log_blocks_per_seg);
351df4b2
JK		 717 }
718 mutex_unlock(&dirty_i->seglist_lock);
b2955550
JK		 719
720 /* send small discards */
2d7b822a 721 list_for_each_entry_safe(entry, this, head, list) {
836b5a63
JK		 722 if (cpc->reason == CP_DISCARD && entry->len < cpc->trim_minlen)
723 goto skip;
37208879 724 f2fs_issue_discard(sbi, entry->blkaddr, entry->len);
f56aa1c5 725 cpc->trimmed += entry->len;
836b5a63 726skip:
b2955550
JK		 727 list_del(&entry->list);
728 SM_I(sbi)->nr_discards -= entry->len;
729 kmem_cache_free(discard_entry_slab, entry);
730 }
351df4b2
JK		 731}
732
184a5cd2 733static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
351df4b2
JK		 734{
735 struct sit_info *sit_i = SIT_I(sbi);
184a5cd2
CY		 736
737 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
351df4b2 738 sit_i->dirty_sentries++;
184a5cd2
CY		 739 return false;
740 }
741
742 return true;
351df4b2
JK		 743}
744
745static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
746 unsigned int segno, int modified)
747{
748 struct seg_entry *se = get_seg_entry(sbi, segno);
749 se->type = type;
750 if (modified)
751 __mark_sit_entry_dirty(sbi, segno);
752}
753
754static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
755{
756 struct seg_entry *se;
757 unsigned int segno, offset;
758 long int new_vblocks;
759
760 segno = GET_SEGNO(sbi, blkaddr);
761
762 se = get_seg_entry(sbi, segno);
763 new_vblocks = se->valid_blocks + del;
491c0854 764 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
351df4b2 765
9850cf4a 766 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) ||
351df4b2
JK		 767 (new_vblocks > sbi->blocks_per_seg)));
768
769 se->valid_blocks = new_vblocks;
770 se->mtime = get_mtime(sbi);
771 SIT_I(sbi)->max_mtime = se->mtime;
772
773 /* Update valid block bitmap */
774 if (del > 0) {
52aca074 775 if (f2fs_test_and_set_bit(offset, se->cur_valid_map))
05796763 776 f2fs_bug_on(sbi, 1);
a66cdd98
JK		 777 if (!f2fs_test_and_set_bit(offset, se->discard_map))
778 sbi->discard_blks--;
351df4b2 779 } else {
52aca074 780 if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map))
05796763 781 f2fs_bug_on(sbi, 1);
a66cdd98
JK		 782 if (f2fs_test_and_clear_bit(offset, se->discard_map))
783 sbi->discard_blks++;
351df4b2
JK		 784 }
785 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
786 se->ckpt_valid_blocks += del;
787
788 __mark_sit_entry_dirty(sbi, segno);
789
790 /* update total number of valid blocks to be written in ckpt area */
791 SIT_I(sbi)->written_valid_blocks += del;
792
793 if (sbi->segs_per_sec > 1)
794 get_sec_entry(sbi, segno)->valid_blocks += del;
795}
796
5e443818 797void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new)
351df4b2 798{
5e443818
JK		 799 update_sit_entry(sbi, new, 1);
800 if (GET_SEGNO(sbi, old) != NULL_SEGNO)
801 update_sit_entry(sbi, old, -1);
802
803 locate_dirty_segment(sbi, GET_SEGNO(sbi, old));
804 locate_dirty_segment(sbi, GET_SEGNO(sbi, new));
351df4b2
JK		 805}
806
807void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
808{
809 unsigned int segno = GET_SEGNO(sbi, addr);
810 struct sit_info *sit_i = SIT_I(sbi);
811
9850cf4a 812 f2fs_bug_on(sbi, addr == NULL_ADDR);
351df4b2
JK		 813 if (addr == NEW_ADDR)
814 return;
815
816 /* add it into sit main buffer */
817 mutex_lock(&sit_i->sentry_lock);
818
819 update_sit_entry(sbi, addr, -1);
820
821 /* add it into dirty seglist */
822 locate_dirty_segment(sbi, segno);
823
824 mutex_unlock(&sit_i->sentry_lock);
825}
826
6e2c64ad
JK		 827bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
828{
829 struct sit_info *sit_i = SIT_I(sbi);
830 unsigned int segno, offset;
831 struct seg_entry *se;
832 bool is_cp = false;
833
834 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR)
835 return true;
836
837 mutex_lock(&sit_i->sentry_lock);
838
839 segno = GET_SEGNO(sbi, blkaddr);
840 se = get_seg_entry(sbi, segno);
841 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
842
843 if (f2fs_test_bit(offset, se->ckpt_valid_map))
844 is_cp = true;
845
846 mutex_unlock(&sit_i->sentry_lock);
847
848 return is_cp;
849}
850
0a8165d7 851/*
351df4b2
JK		 852 * This function should be resided under the curseg_mutex lock
853 */
854static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
e79efe3b 855 struct f2fs_summary *sum)
351df4b2
JK		 856{
857 struct curseg_info *curseg = CURSEG_I(sbi, type);
858 void *addr = curseg->sum_blk;
e79efe3b 859 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
351df4b2 860 memcpy(addr, sum, sizeof(struct f2fs_summary));
351df4b2
JK		 861}
862
0a8165d7 863/*
351df4b2
JK		 864 * Calculate the number of current summary pages for writing
865 */
3fa06d7b 866int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
351df4b2 867{
351df4b2 868 int valid_sum_count = 0;
9a47938b 869 int i, sum_in_page;
351df4b2
JK		 870
871 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
872 if (sbi->ckpt->alloc_type[i] == SSR)
873 valid_sum_count += sbi->blocks_per_seg;
3fa06d7b
CY		 874 else {
875 if (for_ra)
876 valid_sum_count += le16_to_cpu(
877 F2FS_CKPT(sbi)->cur_data_blkoff[i]);
878 else
879 valid_sum_count += curseg_blkoff(sbi, i);
880 }
351df4b2
JK		 881 }
882
9a47938b
FL		 883 sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE -
884 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
885 if (valid_sum_count <= sum_in_page)
351df4b2 886 return 1;
9a47938b
FL		 887 else if ((valid_sum_count - sum_in_page) <=
888 (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
351df4b2
JK		 889 return 2;
890 return 3;
891}
892
0a8165d7 893/*
351df4b2
JK		 894 * Caller should put this summary page
895 */
896struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
897{
898 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
899}
900
381722d2 901void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr)
351df4b2
JK		 902{
903 struct page *page = grab_meta_page(sbi, blk_addr);
381722d2
CY		 904 void *dst = page_address(page);
905
906 if (src)
907 memcpy(dst, src, PAGE_CACHE_SIZE);
908 else
909 memset(dst, 0, PAGE_CACHE_SIZE);
351df4b2
JK		 910 set_page_dirty(page);
911 f2fs_put_page(page, 1);
912}
913
381722d2
CY		 914static void write_sum_page(struct f2fs_sb_info *sbi,
915 struct f2fs_summary_block *sum_blk, block_t blk_addr)
916{
917 update_meta_page(sbi, (void *)sum_blk, blk_addr);
918}
919
60374688
JK		 920static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
921{
922 struct curseg_info *curseg = CURSEG_I(sbi, type);
81fb5e87 923 unsigned int segno = curseg->segno + 1;
60374688
JK		 924 struct free_segmap_info *free_i = FREE_I(sbi);
925
7cd8558b 926 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
81fb5e87 927 return !test_bit(segno, free_i->free_segmap);
60374688
JK		 928 return 0;
929}
930
0a8165d7 931/*
351df4b2
JK		 932 * Find a new segment from the free segments bitmap to right order
933 * This function should be returned with success, otherwise BUG
934 */
935static void get_new_segment(struct f2fs_sb_info *sbi,
936 unsigned int *newseg, bool new_sec, int dir)
937{
938 struct free_segmap_info *free_i = FREE_I(sbi);
351df4b2 939 unsigned int segno, secno, zoneno;
7cd8558b 940 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
351df4b2
JK		 941 unsigned int hint = *newseg / sbi->segs_per_sec;
942 unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
943 unsigned int left_start = hint;
944 bool init = true;
945 int go_left = 0;
946 int i;
947
1a118ccf 948 spin_lock(&free_i->segmap_lock);
351df4b2
JK		 949
950 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
951 segno = find_next_zero_bit(free_i->free_segmap,
0ab14356
CY		 952 (hint + 1) * sbi->segs_per_sec, *newseg + 1);
953 if (segno < (hint + 1) * sbi->segs_per_sec)
351df4b2
JK		 954 goto got_it;
955 }
956find_other_zone:
7cd8558b
JK		 957 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
958 if (secno >= MAIN_SECS(sbi)) {
351df4b2
JK		 959 if (dir == ALLOC_RIGHT) {
960 secno = find_next_zero_bit(free_i->free_secmap,
7cd8558b
JK		 961 MAIN_SECS(sbi), 0);
962 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
351df4b2
JK		 963 } else {
964 go_left = 1;
965 left_start = hint - 1;
966 }
967 }
968 if (go_left == 0)
969 goto skip_left;
970
971 while (test_bit(left_start, free_i->free_secmap)) {
972 if (left_start > 0) {
973 left_start--;
974 continue;
975 }
976 left_start = find_next_zero_bit(free_i->free_secmap,
7cd8558b
JK		 977 MAIN_SECS(sbi), 0);
978 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
351df4b2
JK		 979 break;
980 }
981 secno = left_start;
982skip_left:
983 hint = secno;
984 segno = secno * sbi->segs_per_sec;
985 zoneno = secno / sbi->secs_per_zone;
986
987 /* give up on finding another zone */
988 if (!init)
989 goto got_it;
990 if (sbi->secs_per_zone == 1)
991 goto got_it;
992 if (zoneno == old_zoneno)
993 goto got_it;
994 if (dir == ALLOC_LEFT) {
995 if (!go_left && zoneno + 1 >= total_zones)
996 goto got_it;
997 if (go_left && zoneno == 0)
998 goto got_it;
999 }
1000 for (i = 0; i < NR_CURSEG_TYPE; i++)
1001 if (CURSEG_I(sbi, i)->zone == zoneno)
1002 break;
1003
1004 if (i < NR_CURSEG_TYPE) {
1005 /* zone is in user, try another */
1006 if (go_left)
1007 hint = zoneno * sbi->secs_per_zone - 1;
1008 else if (zoneno + 1 >= total_zones)
1009 hint = 0;
1010 else
1011 hint = (zoneno + 1) * sbi->secs_per_zone;
1012 init = false;
1013 goto find_other_zone;
1014 }
1015got_it:
1016 /* set it as dirty segment in free segmap */
9850cf4a 1017 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
351df4b2
JK		 1018 __set_inuse(sbi, segno);
1019 *newseg = segno;
1a118ccf 1020 spin_unlock(&free_i->segmap_lock);
351df4b2
JK		 1021}
1022
1023static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
1024{
1025 struct curseg_info *curseg = CURSEG_I(sbi, type);
1026 struct summary_footer *sum_footer;
1027
1028 curseg->segno = curseg->next_segno;
1029 curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
1030 curseg->next_blkoff = 0;
1031 curseg->next_segno = NULL_SEGNO;
1032
1033 sum_footer = &(curseg->sum_blk->footer);
1034 memset(sum_footer, 0, sizeof(struct summary_footer));
1035 if (IS_DATASEG(type))
1036 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
1037 if (IS_NODESEG(type))
1038 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
1039 __set_sit_entry_type(sbi, type, curseg->segno, modified);
1040}
1041
0a8165d7 1042/*
351df4b2
JK		 1043 * Allocate a current working segment.
1044 * This function always allocates a free segment in LFS manner.
1045 */
1046static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
1047{
1048 struct curseg_info *curseg = CURSEG_I(sbi, type);
1049 unsigned int segno = curseg->segno;
1050 int dir = ALLOC_LEFT;
1051
1052 write_sum_page(sbi, curseg->sum_blk,
81fb5e87 1053 GET_SUM_BLOCK(sbi, segno));
351df4b2
JK		 1054 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
1055 dir = ALLOC_RIGHT;
1056
1057 if (test_opt(sbi, NOHEAP))
1058 dir = ALLOC_RIGHT;
1059
1060 get_new_segment(sbi, &segno, new_sec, dir);
1061 curseg->next_segno = segno;
1062 reset_curseg(sbi, type, 1);
1063 curseg->alloc_type = LFS;
1064}
1065
1066static void __next_free_blkoff(struct f2fs_sb_info *sbi,
1067 struct curseg_info *seg, block_t start)
1068{
1069 struct seg_entry *se = get_seg_entry(sbi, seg->segno);
e81c93cf 1070 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
60a3b782 1071 unsigned long *target_map = SIT_I(sbi)->tmp_map;
e81c93cf
CL		 1072 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1073 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1074 int i, pos;
1075
1076 for (i = 0; i < entries; i++)
1077 target_map[i] = ckpt_map[i] | cur_map[i];
1078
1079 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
1080
1081 seg->next_blkoff = pos;
351df4b2
JK		 1082}
1083
0a8165d7 1084/*
351df4b2
JK		 1085 * If a segment is written by LFS manner, next block offset is just obtained
1086 * by increasing the current block offset. However, if a segment is written by
1087 * SSR manner, next block offset obtained by calling __next_free_blkoff
1088 */
1089static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
1090 struct curseg_info *seg)
1091{
1092 if (seg->alloc_type == SSR)
1093 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
1094 else
1095 seg->next_blkoff++;
1096}
1097
0a8165d7 1098/*
e1c42045 1099 * This function always allocates a used segment(from dirty seglist) by SSR
351df4b2
JK		 1100 * manner, so it should recover the existing segment information of valid blocks
1101 */
1102static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
1103{
1104 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1105 struct curseg_info *curseg = CURSEG_I(sbi, type);
1106 unsigned int new_segno = curseg->next_segno;
1107 struct f2fs_summary_block *sum_node;
1108 struct page *sum_page;
1109
1110 write_sum_page(sbi, curseg->sum_blk,
1111 GET_SUM_BLOCK(sbi, curseg->segno));
1112 __set_test_and_inuse(sbi, new_segno);
1113
1114 mutex_lock(&dirty_i->seglist_lock);
1115 __remove_dirty_segment(sbi, new_segno, PRE);
1116 __remove_dirty_segment(sbi, new_segno, DIRTY);
1117 mutex_unlock(&dirty_i->seglist_lock);
1118
1119 reset_curseg(sbi, type, 1);
1120 curseg->alloc_type = SSR;
1121 __next_free_blkoff(sbi, curseg, 0);
1122
1123 if (reuse) {
1124 sum_page = get_sum_page(sbi, new_segno);
1125 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
1126 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
1127 f2fs_put_page(sum_page, 1);
1128 }
1129}
1130
43727527
JK		 1131static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
1132{
1133 struct curseg_info *curseg = CURSEG_I(sbi, type);
1134 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
1135
1136 if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0))
1137 return v_ops->get_victim(sbi,
1138 &(curseg)->next_segno, BG_GC, type, SSR);
1139
1140 /* For data segments, let's do SSR more intensively */
1141 for (; type >= CURSEG_HOT_DATA; type--)
1142 if (v_ops->get_victim(sbi, &(curseg)->next_segno,
1143 BG_GC, type, SSR))
1144 return 1;
1145 return 0;
1146}
1147
351df4b2
JK		 1148/*
1149 * flush out current segment and replace it with new segment
1150 * This function should be returned with success, otherwise BUG
1151 */
1152static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
1153 int type, bool force)
1154{
1155 struct curseg_info *curseg = CURSEG_I(sbi, type);
351df4b2 1156
7b405275 1157 if (force)
351df4b2 1158 new_curseg(sbi, type, true);
7b405275 1159 else if (type == CURSEG_WARM_NODE)
351df4b2 1160 new_curseg(sbi, type, false);
60374688
JK		 1161 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
1162 new_curseg(sbi, type, false);
351df4b2
JK		 1163 else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
1164 change_curseg(sbi, type, true);
1165 else
1166 new_curseg(sbi, type, false);
dcdfff65
JK		 1167
1168 stat_inc_seg_type(sbi, curseg);
351df4b2
JK		 1169}
1170
38aa0889
JK		 1171static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type)
1172{
1173 struct curseg_info *curseg = CURSEG_I(sbi, type);
1174 unsigned int old_segno;
1175
1176 old_segno = curseg->segno;
1177 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
1178 locate_dirty_segment(sbi, old_segno);
1179}
1180
351df4b2
JK		 1181void allocate_new_segments(struct f2fs_sb_info *sbi)
1182{
351df4b2
JK		 1183 int i;
1184
38aa0889
JK		 1185 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
1186 __allocate_new_segments(sbi, i);
351df4b2
JK		 1187}
1188
1189static const struct segment_allocation default_salloc_ops = {
1190 .allocate_segment = allocate_segment_by_default,
1191};
1192
4b2fecc8
JK		 1193int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
1194{
f7ef9b83
JK		 1195 __u64 start = F2FS_BYTES_TO_BLK(range->start);
1196 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
4b2fecc8
JK		 1197 unsigned int start_segno, end_segno;
1198 struct cp_control cpc;
c34f42e2 1199 int err = 0;
4b2fecc8 1200
836b5a63 1201 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
4b2fecc8
JK		 1202 return -EINVAL;
1203
9bd27ae4 1204 cpc.trimmed = 0;
7cd8558b 1205 if (end <= MAIN_BLKADDR(sbi))
4b2fecc8
JK		 1206 goto out;
1207
1208 /* start/end segment number in main_area */
7cd8558b
JK		 1209 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
1210 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
1211 GET_SEGNO(sbi, end);
4b2fecc8 1212 cpc.reason = CP_DISCARD;
836b5a63 1213 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
4b2fecc8
JK		 1214
1215 /* do checkpoint to issue discard commands safely */
bba681cb
JK		 1216 for (; start_segno <= end_segno; start_segno = cpc.trim_end + 1) {
1217 cpc.trim_start = start_segno;
a66cdd98
JK		 1218
1219 if (sbi->discard_blks == 0)
1220 break;
1221 else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi))
1222 cpc.trim_end = end_segno;
1223 else
1224 cpc.trim_end = min_t(unsigned int,
1225 rounddown(start_segno +
bba681cb
JK		 1226 BATCHED_TRIM_SEGMENTS(sbi),
1227 sbi->segs_per_sec) - 1, end_segno);
1228
1229 mutex_lock(&sbi->gc_mutex);
c34f42e2 1230 err = write_checkpoint(sbi, &cpc);
bba681cb
JK		 1231 mutex_unlock(&sbi->gc_mutex);
1232 }
4b2fecc8 1233out:
f7ef9b83 1234 range->len = F2FS_BLK_TO_BYTES(cpc.trimmed);
c34f42e2 1235 return err;
4b2fecc8
JK		 1236}
1237
351df4b2
JK		 1238static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
1239{
1240 struct curseg_info *curseg = CURSEG_I(sbi, type);
1241 if (curseg->next_blkoff < sbi->blocks_per_seg)
1242 return true;
1243 return false;
1244}
1245
1246static int __get_segment_type_2(struct page *page, enum page_type p_type)
1247{
1248 if (p_type == DATA)
1249 return CURSEG_HOT_DATA;
1250 else
1251 return CURSEG_HOT_NODE;
1252}
1253
1254static int __get_segment_type_4(struct page *page, enum page_type p_type)
1255{
1256 if (p_type == DATA) {
1257 struct inode *inode = page->mapping->host;
1258
1259 if (S_ISDIR(inode->i_mode))
1260 return CURSEG_HOT_DATA;
1261 else
1262 return CURSEG_COLD_DATA;
1263 } else {
a344b9fd
JK		 1264 if (IS_DNODE(page) && is_cold_node(page))
1265 return CURSEG_WARM_NODE;
351df4b2
JK		 1266 else
1267 return CURSEG_COLD_NODE;
1268 }
1269}
1270
1271static int __get_segment_type_6(struct page *page, enum page_type p_type)
1272{
1273 if (p_type == DATA) {
1274 struct inode *inode = page->mapping->host;
1275
1276 if (S_ISDIR(inode->i_mode))
1277 return CURSEG_HOT_DATA;
354a3399 1278 else if (is_cold_data(page) || file_is_cold(inode))
351df4b2
JK		 1279 return CURSEG_COLD_DATA;
1280 else
1281 return CURSEG_WARM_DATA;
1282 } else {
1283 if (IS_DNODE(page))
1284 return is_cold_node(page) ? CURSEG_WARM_NODE :
1285 CURSEG_HOT_NODE;
1286 else
1287 return CURSEG_COLD_NODE;
1288 }
1289}
1290
1291static int __get_segment_type(struct page *page, enum page_type p_type)
1292{
4081363f 1293 switch (F2FS_P_SB(page)->active_logs) {
351df4b2
JK		 1294 case 2:
1295 return __get_segment_type_2(page, p_type);
1296 case 4:
1297 return __get_segment_type_4(page, p_type);
351df4b2 1298 }
12a67146 1299 /* NR_CURSEG_TYPE(6) logs by default */
9850cf4a
JK		 1300 f2fs_bug_on(F2FS_P_SB(page),
1301 F2FS_P_SB(page)->active_logs != NR_CURSEG_TYPE);
12a67146 1302 return __get_segment_type_6(page, p_type);
351df4b2
JK		 1303}
1304
bfad7c2d
JK		 1305void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
1306 block_t old_blkaddr, block_t *new_blkaddr,
1307 struct f2fs_summary *sum, int type)
351df4b2
JK		 1308{
1309 struct sit_info *sit_i = SIT_I(sbi);
1310 struct curseg_info *curseg;
38aa0889
JK		 1311 bool direct_io = (type == CURSEG_DIRECT_IO);
1312
1313 type = direct_io ? CURSEG_WARM_DATA : type;
351df4b2 1314
351df4b2
JK		 1315 curseg = CURSEG_I(sbi, type);
1316
1317 mutex_lock(&curseg->curseg_mutex);
21cb1d99 1318 mutex_lock(&sit_i->sentry_lock);
351df4b2 1319
38aa0889 1320 /* direct_io'ed data is aligned to the segment for better performance */
47e70ca4
JK		 1321 if (direct_io && curseg->next_blkoff &&
1322 !has_not_enough_free_secs(sbi, 0))
38aa0889
JK		 1323 __allocate_new_segments(sbi, type);
1324
351df4b2 1325 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
351df4b2
JK		 1326
1327 /*
1328 * __add_sum_entry should be resided under the curseg_mutex
1329 * because, this function updates a summary entry in the
1330 * current summary block.
1331 */
e79efe3b 1332 __add_sum_entry(sbi, type, sum);
351df4b2 1333
351df4b2 1334 __refresh_next_blkoff(sbi, curseg);
dcdfff65
JK		 1335
1336 stat_inc_block_count(sbi, curseg);
351df4b2 1337
5e443818
JK		 1338 if (!__has_curseg_space(sbi, type))
1339 sit_i->s_ops->allocate_segment(sbi, type, false);
351df4b2
JK		 1340 /*
1341 * SIT information should be updated before segment allocation,
1342 * since SSR needs latest valid block information.
1343 */
1344 refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
5e443818 1345
351df4b2
JK		 1346 mutex_unlock(&sit_i->sentry_lock);
1347
bfad7c2d 1348 if (page && IS_NODESEG(type))
351df4b2
JK		 1349 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
1350
bfad7c2d
JK		 1351 mutex_unlock(&curseg->curseg_mutex);
1352}
1353
05ca3632 1354static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
bfad7c2d 1355{
05ca3632 1356 int type = __get_segment_type(fio->page, fio->type);
bfad7c2d 1357
05ca3632
JK		 1358 allocate_data_block(fio->sbi, fio->page, fio->blk_addr,
1359 &fio->blk_addr, sum, type);
bfad7c2d 1360
351df4b2 1361 /* writeout dirty page into bdev */
05ca3632 1362 f2fs_submit_page_mbio(fio);
351df4b2
JK		 1363}
1364
577e3495 1365void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
351df4b2 1366{
458e6197 1367 struct f2fs_io_info fio = {
05ca3632 1368 .sbi = sbi,
458e6197 1369 .type = META,
cf04e8eb
JK		 1370 .rw = WRITE_SYNC | REQ_META | REQ_PRIO,
1371 .blk_addr = page->index,
05ca3632 1372 .page = page,
4375a336 1373 .encrypted_page = NULL,
458e6197
JK		 1374 };
1375
2b947003
CY		 1376 if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
1377 fio.rw &= ~REQ_META;
1378
351df4b2 1379 set_page_writeback(page);
05ca3632 1380 f2fs_submit_page_mbio(&fio);
351df4b2
JK		 1381}
1382
05ca3632 1383void write_node_page(unsigned int nid, struct f2fs_io_info *fio)
351df4b2
JK		 1384{
1385 struct f2fs_summary sum;
05ca3632 1386
351df4b2 1387 set_summary(&sum, nid, 0, 0);
05ca3632 1388 do_write_page(&sum, fio);
351df4b2
JK		 1389}
1390
05ca3632 1391void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio)
351df4b2 1392{
05ca3632 1393 struct f2fs_sb_info *sbi = fio->sbi;
351df4b2
JK		 1394 struct f2fs_summary sum;
1395 struct node_info ni;
1396
9850cf4a 1397 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
351df4b2
JK		 1398 get_node_info(sbi, dn->nid, &ni);
1399 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
05ca3632 1400 do_write_page(&sum, fio);
e1509cf2 1401 dn->data_blkaddr = fio->blk_addr;
351df4b2
JK		 1402}
1403
05ca3632 1404void rewrite_data_page(struct f2fs_io_info *fio)
351df4b2 1405{
05ca3632
JK		 1406 stat_inc_inplace_blocks(fio->sbi);
1407 f2fs_submit_page_mbio(fio);
351df4b2
JK		 1408}
1409
528e3459
CY		 1410static void __f2fs_replace_block(struct f2fs_sb_info *sbi,
1411 struct f2fs_summary *sum,
19f106bc 1412 block_t old_blkaddr, block_t new_blkaddr,
28bc106b 1413 bool recover_curseg, bool recover_newaddr)
351df4b2
JK		 1414{
1415 struct sit_info *sit_i = SIT_I(sbi);
1416 struct curseg_info *curseg;
1417 unsigned int segno, old_cursegno;
1418 struct seg_entry *se;
1419 int type;
19f106bc 1420 unsigned short old_blkoff;
351df4b2
JK		 1421
1422 segno = GET_SEGNO(sbi, new_blkaddr);
1423 se = get_seg_entry(sbi, segno);
1424 type = se->type;
1425
19f106bc
CY		 1426 if (!recover_curseg) {
1427 /* for recovery flow */
1428 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
1429 if (old_blkaddr == NULL_ADDR)
1430 type = CURSEG_COLD_DATA;
1431 else
1432 type = CURSEG_WARM_DATA;
1433 }
1434 } else {
1435 if (!IS_CURSEG(sbi, segno))
351df4b2
JK		 1436 type = CURSEG_WARM_DATA;
1437 }
19f106bc 1438
351df4b2
JK		 1439 curseg = CURSEG_I(sbi, type);
1440
1441 mutex_lock(&curseg->curseg_mutex);
1442 mutex_lock(&sit_i->sentry_lock);
1443
1444 old_cursegno = curseg->segno;
19f106bc 1445 old_blkoff = curseg->next_blkoff;
351df4b2
JK		 1446
1447 /* change the current segment */
1448 if (segno != curseg->segno) {
1449 curseg->next_segno = segno;
1450 change_curseg(sbi, type, true);
1451 }
1452
491c0854 1453 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
e79efe3b 1454 __add_sum_entry(sbi, type, sum);
351df4b2 1455
28bc106b 1456 if (!recover_curseg || recover_newaddr)
6e2c64ad
JK		 1457 update_sit_entry(sbi, new_blkaddr, 1);
1458 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1459 update_sit_entry(sbi, old_blkaddr, -1);
1460
1461 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
1462 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
1463
351df4b2 1464 locate_dirty_segment(sbi, old_cursegno);
351df4b2 1465
19f106bc
CY		 1466 if (recover_curseg) {
1467 if (old_cursegno != curseg->segno) {
1468 curseg->next_segno = old_cursegno;
1469 change_curseg(sbi, type, true);
1470 }
1471 curseg->next_blkoff = old_blkoff;
1472 }
1473
351df4b2
JK		 1474 mutex_unlock(&sit_i->sentry_lock);
1475 mutex_unlock(&curseg->curseg_mutex);
1476}
1477
528e3459
CY		 1478void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
1479 block_t old_addr, block_t new_addr,
28bc106b
CY		 1480 unsigned char version, bool recover_curseg,
1481 bool recover_newaddr)
528e3459
CY		 1482{
1483 struct f2fs_summary sum;
1484
1485 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
1486
28bc106b
CY		 1487 __f2fs_replace_block(sbi, &sum, old_addr, new_addr,
1488 recover_curseg, recover_newaddr);
528e3459
CY		 1489
1490 dn->data_blkaddr = new_addr;
1491 set_data_blkaddr(dn);
1492 f2fs_update_extent_cache(dn);
1493}
1494
93dfe2ac 1495void f2fs_wait_on_page_writeback(struct page *page,
fec1d657 1496 enum page_type type, bool ordered)
93dfe2ac 1497{
93dfe2ac 1498 if (PageWriteback(page)) {
4081363f
JK		 1499 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
1500
0c3a5797 1501 f2fs_submit_merged_bio_cond(sbi, NULL, page, 0, type, WRITE);
fec1d657
JK		 1502 if (ordered)
1503 wait_on_page_writeback(page);
1504 else
1505 wait_for_stable_page(page);
93dfe2ac
JK		 1506 }
1507}
1508
08b39fbd
CY		 1509void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi,
1510 block_t blkaddr)
1511{
1512 struct page *cpage;
1513
1514 if (blkaddr == NEW_ADDR)
1515 return;
1516
1517 f2fs_bug_on(sbi, blkaddr == NULL_ADDR);
1518
1519 cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
1520 if (cpage) {
fec1d657 1521 f2fs_wait_on_page_writeback(cpage, DATA, true);
08b39fbd
CY		 1522 f2fs_put_page(cpage, 1);
1523 }
1524}
1525
351df4b2
JK		 1526static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1527{
1528 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1529 struct curseg_info *seg_i;
1530 unsigned char *kaddr;
1531 struct page *page;
1532 block_t start;
1533 int i, j, offset;
1534
1535 start = start_sum_block(sbi);
1536
1537 page = get_meta_page(sbi, start++);
1538 kaddr = (unsigned char *)page_address(page);
1539
1540 /* Step 1: restore nat cache */
1541 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1542 memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE);
1543
1544 /* Step 2: restore sit cache */
1545 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1546 memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE,
1547 SUM_JOURNAL_SIZE);
1548 offset = 2 * SUM_JOURNAL_SIZE;
1549
1550 /* Step 3: restore summary entries */
1551 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1552 unsigned short blk_off;
1553 unsigned int segno;
1554
1555 seg_i = CURSEG_I(sbi, i);
1556 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1557 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1558 seg_i->next_segno = segno;
1559 reset_curseg(sbi, i, 0);
1560 seg_i->alloc_type = ckpt->alloc_type[i];
1561 seg_i->next_blkoff = blk_off;
1562
1563 if (seg_i->alloc_type == SSR)
1564 blk_off = sbi->blocks_per_seg;
1565
1566 for (j = 0; j < blk_off; j++) {
1567 struct f2fs_summary *s;
1568 s = (struct f2fs_summary *)(kaddr + offset);
1569 seg_i->sum_blk->entries[j] = *s;
1570 offset += SUMMARY_SIZE;
1571 if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1572 SUM_FOOTER_SIZE)
1573 continue;
1574
1575 f2fs_put_page(page, 1);
1576 page = NULL;
1577
1578 page = get_meta_page(sbi, start++);
1579 kaddr = (unsigned char *)page_address(page);
1580 offset = 0;
1581 }
1582 }
1583 f2fs_put_page(page, 1);
1584 return 0;
1585}
1586
1587static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1588{
1589 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1590 struct f2fs_summary_block *sum;
1591 struct curseg_info *curseg;
1592 struct page *new;
1593 unsigned short blk_off;
1594 unsigned int segno = 0;
1595 block_t blk_addr = 0;
1596
1597 /* get segment number and block addr */
1598 if (IS_DATASEG(type)) {
1599 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1600 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1601 CURSEG_HOT_DATA]);
119ee914 1602 if (__exist_node_summaries(sbi))
351df4b2
JK		 1603 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1604 else
1605 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1606 } else {
1607 segno = le32_to_cpu(ckpt->cur_node_segno[type -
1608 CURSEG_HOT_NODE]);
1609 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1610 CURSEG_HOT_NODE]);
119ee914 1611 if (__exist_node_summaries(sbi))
351df4b2
JK		 1612 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1613 type - CURSEG_HOT_NODE);
1614 else
1615 blk_addr = GET_SUM_BLOCK(sbi, segno);
1616 }
1617
1618 new = get_meta_page(sbi, blk_addr);
1619 sum = (struct f2fs_summary_block *)page_address(new);
1620
1621 if (IS_NODESEG(type)) {
119ee914 1622 if (__exist_node_summaries(sbi)) {
351df4b2
JK		 1623 struct f2fs_summary *ns = &sum->entries[0];
1624 int i;
1625 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1626 ns->version = 0;
1627 ns->ofs_in_node = 0;
1628 }
1629 } else {
d653788a
GZ		 1630 int err;
1631
1632 err = restore_node_summary(sbi, segno, sum);
1633 if (err) {
351df4b2 1634 f2fs_put_page(new, 1);
d653788a 1635 return err;
351df4b2
JK		 1636 }
1637 }
1638 }
1639
1640 /* set uncompleted segment to curseg */
1641 curseg = CURSEG_I(sbi, type);
1642 mutex_lock(&curseg->curseg_mutex);
1643 memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE);
1644 curseg->next_segno = segno;
1645 reset_curseg(sbi, type, 0);
1646 curseg->alloc_type = ckpt->alloc_type[type];
1647 curseg->next_blkoff = blk_off;
1648 mutex_unlock(&curseg->curseg_mutex);
1649 f2fs_put_page(new, 1);
1650 return 0;
1651}
1652
1653static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1654{
1655 int type = CURSEG_HOT_DATA;
e4fc5fbf 1656 int err;
351df4b2 1657
25ca923b 1658 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {
3fa06d7b
CY		 1659 int npages = npages_for_summary_flush(sbi, true);
1660
1661 if (npages >= 2)
1662 ra_meta_pages(sbi, start_sum_block(sbi), npages,
26879fb1 1663 META_CP, true);
3fa06d7b 1664
351df4b2
JK		 1665 /* restore for compacted data summary */
1666 if (read_compacted_summaries(sbi))
1667 return -EINVAL;
1668 type = CURSEG_HOT_NODE;
1669 }
1670
119ee914 1671 if (__exist_node_summaries(sbi))
3fa06d7b 1672 ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
26879fb1 1673 NR_CURSEG_TYPE - type, META_CP, true);
3fa06d7b 1674
e4fc5fbf
CY		 1675 for (; type <= CURSEG_COLD_NODE; type++) {
1676 err = read_normal_summaries(sbi, type);
1677 if (err)
1678 return err;
1679 }
1680
351df4b2
JK		 1681 return 0;
1682}
1683
1684static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1685{
1686 struct page *page;
1687 unsigned char *kaddr;
1688 struct f2fs_summary *summary;
1689 struct curseg_info *seg_i;
1690 int written_size = 0;
1691 int i, j;
1692
1693 page = grab_meta_page(sbi, blkaddr++);
1694 kaddr = (unsigned char *)page_address(page);
1695
1696 /* Step 1: write nat cache */
1697 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1698 memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE);
1699 written_size += SUM_JOURNAL_SIZE;
1700
1701 /* Step 2: write sit cache */
1702 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1703 memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits,
1704 SUM_JOURNAL_SIZE);
1705 written_size += SUM_JOURNAL_SIZE;
1706
351df4b2
JK		 1707 /* Step 3: write summary entries */
1708 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1709 unsigned short blkoff;
1710 seg_i = CURSEG_I(sbi, i);
1711 if (sbi->ckpt->alloc_type[i] == SSR)
1712 blkoff = sbi->blocks_per_seg;
1713 else
1714 blkoff = curseg_blkoff(sbi, i);
1715
1716 for (j = 0; j < blkoff; j++) {
1717 if (!page) {
1718 page = grab_meta_page(sbi, blkaddr++);
1719 kaddr = (unsigned char *)page_address(page);
1720 written_size = 0;
1721 }
1722 summary = (struct f2fs_summary *)(kaddr + written_size);
1723 *summary = seg_i->sum_blk->entries[j];
1724 written_size += SUMMARY_SIZE;
351df4b2
JK		 1725
1726 if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1727 SUM_FOOTER_SIZE)
1728 continue;
1729
e8d61a74 1730 set_page_dirty(page);
351df4b2
JK		 1731 f2fs_put_page(page, 1);
1732 page = NULL;
1733 }
1734 }
e8d61a74
CY		 1735 if (page) {
1736 set_page_dirty(page);
351df4b2 1737 f2fs_put_page(page, 1);
e8d61a74 1738 }
351df4b2
JK		 1739}
1740
1741static void write_normal_summaries(struct f2fs_sb_info *sbi,
1742 block_t blkaddr, int type)
1743{
1744 int i, end;
1745 if (IS_DATASEG(type))
1746 end = type + NR_CURSEG_DATA_TYPE;
1747 else
1748 end = type + NR_CURSEG_NODE_TYPE;
1749
1750 for (i = type; i < end; i++) {
1751 struct curseg_info *sum = CURSEG_I(sbi, i);
1752 mutex_lock(&sum->curseg_mutex);
1753 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type));
1754 mutex_unlock(&sum->curseg_mutex);
1755 }
1756}
1757
1758void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1759{
25ca923b 1760 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))
351df4b2
JK		 1761 write_compacted_summaries(sbi, start_blk);
1762 else
1763 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
1764}
1765
1766void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1767{
119ee914 1768 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
351df4b2
JK		 1769}
1770
1771int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type,
1772 unsigned int val, int alloc)
1773{
1774 int i;
1775
1776 if (type == NAT_JOURNAL) {
1777 for (i = 0; i < nats_in_cursum(sum); i++) {
1778 if (le32_to_cpu(nid_in_journal(sum, i)) == val)
1779 return i;
1780 }
855639de 1781 if (alloc && __has_cursum_space(sum, 1, NAT_JOURNAL))
351df4b2
JK		 1782 return update_nats_in_cursum(sum, 1);
1783 } else if (type == SIT_JOURNAL) {
1784 for (i = 0; i < sits_in_cursum(sum); i++)
1785 if (le32_to_cpu(segno_in_journal(sum, i)) == val)
1786 return i;
855639de 1787 if (alloc && __has_cursum_space(sum, 1, SIT_JOURNAL))
351df4b2
JK		 1788 return update_sits_in_cursum(sum, 1);
1789 }
1790 return -1;
1791}
1792
1793static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
1794 unsigned int segno)
1795{
2cc22186 1796 return get_meta_page(sbi, current_sit_addr(sbi, segno));
351df4b2
JK		 1797}
1798
1799static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
1800 unsigned int start)
1801{
1802 struct sit_info *sit_i = SIT_I(sbi);
1803 struct page *src_page, *dst_page;
1804 pgoff_t src_off, dst_off;
1805 void *src_addr, *dst_addr;
1806
1807 src_off = current_sit_addr(sbi, start);
1808 dst_off = next_sit_addr(sbi, src_off);
1809
1810 /* get current sit block page without lock */
1811 src_page = get_meta_page(sbi, src_off);
1812 dst_page = grab_meta_page(sbi, dst_off);
9850cf4a 1813 f2fs_bug_on(sbi, PageDirty(src_page));
351df4b2
JK		 1814
1815 src_addr = page_address(src_page);
1816 dst_addr = page_address(dst_page);
1817 memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
1818
1819 set_page_dirty(dst_page);
1820 f2fs_put_page(src_page, 1);
1821
1822 set_to_next_sit(sit_i, start);
1823
1824 return dst_page;
1825}
1826
184a5cd2
CY		 1827static struct sit_entry_set *grab_sit_entry_set(void)
1828{
1829 struct sit_entry_set *ses =
80c54505 1830 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
184a5cd2
CY		 1831
1832 ses->entry_cnt = 0;
1833 INIT_LIST_HEAD(&ses->set_list);
1834 return ses;
1835}
1836
1837static void release_sit_entry_set(struct sit_entry_set *ses)
1838{
1839 list_del(&ses->set_list);
1840 kmem_cache_free(sit_entry_set_slab, ses);
1841}
1842
1843static void adjust_sit_entry_set(struct sit_entry_set *ses,
1844 struct list_head *head)
1845{
1846 struct sit_entry_set *next = ses;
1847
1848 if (list_is_last(&ses->set_list, head))
1849 return;
1850
1851 list_for_each_entry_continue(next, head, set_list)
1852 if (ses->entry_cnt <= next->entry_cnt)
1853 break;
1854
1855 list_move_tail(&ses->set_list, &next->set_list);
1856}
1857
1858static void add_sit_entry(unsigned int segno, struct list_head *head)
1859{
1860 struct sit_entry_set *ses;
1861 unsigned int start_segno = START_SEGNO(segno);
1862
1863 list_for_each_entry(ses, head, set_list) {
1864 if (ses->start_segno == start_segno) {
1865 ses->entry_cnt++;
1866 adjust_sit_entry_set(ses, head);
1867 return;
1868 }
1869 }
1870
1871 ses = grab_sit_entry_set();
1872
1873 ses->start_segno = start_segno;
1874 ses->entry_cnt++;
1875 list_add(&ses->set_list, head);
1876}
1877
1878static void add_sits_in_set(struct f2fs_sb_info *sbi)
1879{
1880 struct f2fs_sm_info *sm_info = SM_I(sbi);
1881 struct list_head *set_list = &sm_info->sit_entry_set;
1882 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
184a5cd2
CY		 1883 unsigned int segno;
1884
7cd8558b 1885 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
184a5cd2
CY		 1886 add_sit_entry(segno, set_list);
1887}
1888
1889static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
351df4b2
JK		 1890{
1891 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1892 struct f2fs_summary_block *sum = curseg->sum_blk;
1893 int i;
1894
184a5cd2
CY		 1895 for (i = sits_in_cursum(sum) - 1; i >= 0; i--) {
1896 unsigned int segno;
1897 bool dirtied;
1898
1899 segno = le32_to_cpu(segno_in_journal(sum, i));
1900 dirtied = __mark_sit_entry_dirty(sbi, segno);
1901
1902 if (!dirtied)
1903 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
351df4b2 1904 }
184a5cd2 1905 update_sits_in_cursum(sum, -sits_in_cursum(sum));
351df4b2
JK		 1906}
1907
0a8165d7 1908/*
351df4b2
JK		 1909 * CP calls this function, which flushes SIT entries including sit_journal,
1910 * and moves prefree segs to free segs.
1911 */
4b2fecc8 1912void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
351df4b2
JK		 1913{
1914 struct sit_info *sit_i = SIT_I(sbi);
1915 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
1916 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1917 struct f2fs_summary_block *sum = curseg->sum_blk;
184a5cd2
CY		 1918 struct sit_entry_set *ses, *tmp;
1919 struct list_head *head = &SM_I(sbi)->sit_entry_set;
184a5cd2 1920 bool to_journal = true;
4b2fecc8 1921 struct seg_entry *se;
351df4b2
JK		 1922
1923 mutex_lock(&curseg->curseg_mutex);
1924 mutex_lock(&sit_i->sentry_lock);
1925
2b11a74b
WL		 1926 if (!sit_i->dirty_sentries)
1927 goto out;
1928
351df4b2 1929 /*
184a5cd2
CY		 1930 * add and account sit entries of dirty bitmap in sit entry
1931 * set temporarily
351df4b2 1932 */
184a5cd2 1933 add_sits_in_set(sbi);
351df4b2 1934
184a5cd2
CY		 1935 /*
1936 * if there are no enough space in journal to store dirty sit
1937 * entries, remove all entries from journal and add and account
1938 * them in sit entry set.
1939 */
1940 if (!__has_cursum_space(sum, sit_i->dirty_sentries, SIT_JOURNAL))
1941 remove_sits_in_journal(sbi);
b2955550 1942
184a5cd2
CY		 1943 /*
1944 * there are two steps to flush sit entries:
1945 * #1, flush sit entries to journal in current cold data summary block.
1946 * #2, flush sit entries to sit page.
1947 */
1948 list_for_each_entry_safe(ses, tmp, head, set_list) {
4a257ed6 1949 struct page *page = NULL;
184a5cd2
CY		 1950 struct f2fs_sit_block *raw_sit = NULL;
1951 unsigned int start_segno = ses->start_segno;
1952 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
7cd8558b 1953 (unsigned long)MAIN_SEGS(sbi));
184a5cd2
CY		 1954 unsigned int segno = start_segno;
1955
1956 if (to_journal &&
1957 !__has_cursum_space(sum, ses->entry_cnt, SIT_JOURNAL))
1958 to_journal = false;
1959
1960 if (!to_journal) {
1961 page = get_next_sit_page(sbi, start_segno);
1962 raw_sit = page_address(page);
351df4b2 1963 }
351df4b2 1964
184a5cd2
CY		 1965 /* flush dirty sit entries in region of current sit set */
1966 for_each_set_bit_from(segno, bitmap, end) {
1967 int offset, sit_offset;
4b2fecc8
JK		 1968
1969 se = get_seg_entry(sbi, segno);
184a5cd2
CY		 1970
1971 /* add discard candidates */
d7bc2484 1972 if (cpc->reason != CP_DISCARD) {
4b2fecc8
JK		 1973 cpc->trim_start = segno;
1974 add_discard_addrs(sbi, cpc);
1975 }
184a5cd2
CY		 1976
1977 if (to_journal) {
1978 offset = lookup_journal_in_cursum(sum,
1979 SIT_JOURNAL, segno, 1);
1980 f2fs_bug_on(sbi, offset < 0);
1981 segno_in_journal(sum, offset) =
1982 cpu_to_le32(segno);
1983 seg_info_to_raw_sit(se,
1984 &sit_in_journal(sum, offset));
1985 } else {
1986 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
1987 seg_info_to_raw_sit(se,
1988 &raw_sit->entries[sit_offset]);
1989 }
351df4b2 1990
184a5cd2
CY		 1991 __clear_bit(segno, bitmap);
1992 sit_i->dirty_sentries--;
1993 ses->entry_cnt--;
351df4b2
JK		 1994 }
1995
184a5cd2
CY		 1996 if (!to_journal)
1997 f2fs_put_page(page, 1);
1998
1999 f2fs_bug_on(sbi, ses->entry_cnt);
2000 release_sit_entry_set(ses);
351df4b2 2001 }
184a5cd2
CY		 2002
2003 f2fs_bug_on(sbi, !list_empty(head));
2004 f2fs_bug_on(sbi, sit_i->dirty_sentries);
184a5cd2 2005out:
4b2fecc8
JK		 2006 if (cpc->reason == CP_DISCARD) {
2007 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
2008 add_discard_addrs(sbi, cpc);
2009 }
351df4b2
JK		 2010 mutex_unlock(&sit_i->sentry_lock);
2011 mutex_unlock(&curseg->curseg_mutex);
2012
351df4b2
JK		 2013 set_prefree_as_free_segments(sbi);
2014}
2015
2016static int build_sit_info(struct f2fs_sb_info *sbi)
2017{
2018 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2019 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2020 struct sit_info *sit_i;
2021 unsigned int sit_segs, start;
2022 char *src_bitmap, *dst_bitmap;
2023 unsigned int bitmap_size;
2024
2025 /* allocate memory for SIT information */
2026 sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
2027 if (!sit_i)
2028 return -ENOMEM;
2029
2030 SM_I(sbi)->sit_info = sit_i;
2031
39307a8e
JK		 2032 sit_i->sentries = f2fs_kvzalloc(MAIN_SEGS(sbi) *
2033 sizeof(struct seg_entry), GFP_KERNEL);
351df4b2
JK		 2034 if (!sit_i->sentries)
2035 return -ENOMEM;
2036
7cd8558b 2037 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
39307a8e 2038 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
351df4b2
JK		 2039 if (!sit_i->dirty_sentries_bitmap)
2040 return -ENOMEM;
2041
7cd8558b 2042 for (start = 0; start < MAIN_SEGS(sbi); start++) {
351df4b2
JK		 2043 sit_i->sentries[start].cur_valid_map
2044 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2045 sit_i->sentries[start].ckpt_valid_map
2046 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
a66cdd98
JK		 2047 sit_i->sentries[start].discard_map
2048 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2049 if (!sit_i->sentries[start].cur_valid_map ||
2050 !sit_i->sentries[start].ckpt_valid_map ||
2051 !sit_i->sentries[start].discard_map)
351df4b2
JK		 2052 return -ENOMEM;
2053 }
2054
60a3b782
JK		 2055 sit_i->tmp_map = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2056 if (!sit_i->tmp_map)
2057 return -ENOMEM;
2058
351df4b2 2059 if (sbi->segs_per_sec > 1) {
39307a8e
JK		 2060 sit_i->sec_entries = f2fs_kvzalloc(MAIN_SECS(sbi) *
2061 sizeof(struct sec_entry), GFP_KERNEL);
351df4b2
JK		 2062 if (!sit_i->sec_entries)
2063 return -ENOMEM;
2064 }
2065
2066 /* get information related with SIT */
2067 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
2068
2069 /* setup SIT bitmap from ckeckpoint pack */
2070 bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
2071 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
2072
79b5793b 2073 dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
351df4b2
JK		 2074 if (!dst_bitmap)
2075 return -ENOMEM;
351df4b2
JK		 2076
2077 /* init SIT information */
2078 sit_i->s_ops = &default_salloc_ops;
2079
2080 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
2081 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
2082 sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
2083 sit_i->sit_bitmap = dst_bitmap;
2084 sit_i->bitmap_size = bitmap_size;
2085 sit_i->dirty_sentries = 0;
2086 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
2087 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
2088 sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
2089 mutex_init(&sit_i->sentry_lock);
2090 return 0;
2091}
2092
2093static int build_free_segmap(struct f2fs_sb_info *sbi)
2094{
351df4b2
JK		 2095 struct free_segmap_info *free_i;
2096 unsigned int bitmap_size, sec_bitmap_size;
2097
2098 /* allocate memory for free segmap information */
2099 free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
2100 if (!free_i)
2101 return -ENOMEM;
2102
2103 SM_I(sbi)->free_info = free_i;
2104
7cd8558b 2105 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
39307a8e 2106 free_i->free_segmap = f2fs_kvmalloc(bitmap_size, GFP_KERNEL);
351df4b2
JK		 2107 if (!free_i->free_segmap)
2108 return -ENOMEM;
2109
7cd8558b 2110 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
39307a8e 2111 free_i->free_secmap = f2fs_kvmalloc(sec_bitmap_size, GFP_KERNEL);
351df4b2
JK		 2112 if (!free_i->free_secmap)
2113 return -ENOMEM;
2114
2115 /* set all segments as dirty temporarily */
2116 memset(free_i->free_segmap, 0xff, bitmap_size);
2117 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
2118
2119 /* init free segmap information */
7cd8558b 2120 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
351df4b2
JK		 2121 free_i->free_segments = 0;
2122 free_i->free_sections = 0;
1a118ccf 2123 spin_lock_init(&free_i->segmap_lock);
351df4b2
JK		 2124 return 0;
2125}
2126
2127static int build_curseg(struct f2fs_sb_info *sbi)
2128{
1042d60f 2129 struct curseg_info *array;
351df4b2
JK		 2130 int i;
2131
b434babf 2132 array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL);
351df4b2
JK		 2133 if (!array)
2134 return -ENOMEM;
2135
2136 SM_I(sbi)->curseg_array = array;
2137
2138 for (i = 0; i < NR_CURSEG_TYPE; i++) {
2139 mutex_init(&array[i].curseg_mutex);
2140 array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL);
2141 if (!array[i].sum_blk)
2142 return -ENOMEM;
2143 array[i].segno = NULL_SEGNO;
2144 array[i].next_blkoff = 0;
2145 }
2146 return restore_curseg_summaries(sbi);
2147}
2148
2149static void build_sit_entries(struct f2fs_sb_info *sbi)
2150{
2151 struct sit_info *sit_i = SIT_I(sbi);
2152 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
2153 struct f2fs_summary_block *sum = curseg->sum_blk;
74de593a
CY		 2154 int sit_blk_cnt = SIT_BLK_CNT(sbi);
2155 unsigned int i, start, end;
2156 unsigned int readed, start_blk = 0;
90a893c7 2157 int nrpages = MAX_BIO_BLOCKS(sbi);
351df4b2 2158
74de593a 2159 do {
26879fb1 2160 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true);
74de593a
CY		 2161
2162 start = start_blk * sit_i->sents_per_block;
2163 end = (start_blk + readed) * sit_i->sents_per_block;
2164
7cd8558b 2165 for (; start < end && start < MAIN_SEGS(sbi); start++) {
74de593a
CY		 2166 struct seg_entry *se = &sit_i->sentries[start];
2167 struct f2fs_sit_block *sit_blk;
2168 struct f2fs_sit_entry sit;
2169 struct page *page;
2170
2171 mutex_lock(&curseg->curseg_mutex);
2172 for (i = 0; i < sits_in_cursum(sum); i++) {
6c311ec6
CF		 2173 if (le32_to_cpu(segno_in_journal(sum, i))
2174 == start) {
74de593a
CY		 2175 sit = sit_in_journal(sum, i);
2176 mutex_unlock(&curseg->curseg_mutex);
2177 goto got_it;
2178 }
351df4b2 2179 }
74de593a
CY		 2180 mutex_unlock(&curseg->curseg_mutex);
2181
2182 page = get_current_sit_page(sbi, start);
2183 sit_blk = (struct f2fs_sit_block *)page_address(page);
2184 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
2185 f2fs_put_page(page, 1);
351df4b2 2186got_it:
74de593a
CY		 2187 check_block_count(sbi, start, &sit);
2188 seg_info_from_raw_sit(se, &sit);
a66cdd98
JK		 2189
2190 /* build discard map only one time */
2191 memcpy(se->discard_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE);
2192 sbi->discard_blks += sbi->blocks_per_seg - se->valid_blocks;
2193
74de593a
CY		 2194 if (sbi->segs_per_sec > 1) {
2195 struct sec_entry *e = get_sec_entry(sbi, start);
2196 e->valid_blocks += se->valid_blocks;
2197 }
351df4b2 2198 }
74de593a
CY		 2199 start_blk += readed;
2200 } while (start_blk < sit_blk_cnt);
351df4b2
JK		 2201}
2202
2203static void init_free_segmap(struct f2fs_sb_info *sbi)
2204{
2205 unsigned int start;
2206 int type;
2207
7cd8558b 2208 for (start = 0; start < MAIN_SEGS(sbi); start++) {
351df4b2
JK		 2209 struct seg_entry *sentry = get_seg_entry(sbi, start);
2210 if (!sentry->valid_blocks)
2211 __set_free(sbi, start);
2212 }
2213
2214 /* set use the current segments */
2215 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
2216 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
2217 __set_test_and_inuse(sbi, curseg_t->segno);
2218 }
2219}
2220
2221static void init_dirty_segmap(struct f2fs_sb_info *sbi)
2222{
2223 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2224 struct free_segmap_info *free_i = FREE_I(sbi);
7cd8558b 2225 unsigned int segno = 0, offset = 0;
351df4b2
JK		 2226 unsigned short valid_blocks;
2227
8736fbf0 2228 while (1) {
351df4b2 2229 /* find dirty segment based on free segmap */
7cd8558b
JK		 2230 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
2231 if (segno >= MAIN_SEGS(sbi))
351df4b2
JK		 2232 break;
2233 offset = segno + 1;
2234 valid_blocks = get_valid_blocks(sbi, segno, 0);
ec325b52 2235 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks)
351df4b2 2236 continue;
ec325b52
JK		 2237 if (valid_blocks > sbi->blocks_per_seg) {
2238 f2fs_bug_on(sbi, 1);
2239 continue;
2240 }
351df4b2
JK		 2241 mutex_lock(&dirty_i->seglist_lock);
2242 __locate_dirty_segment(sbi, segno, DIRTY);
2243 mutex_unlock(&dirty_i->seglist_lock);
2244 }
2245}
2246
5ec4e49f 2247static int init_victim_secmap(struct f2fs_sb_info *sbi)
351df4b2
JK		 2248{
2249 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
7cd8558b 2250 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
351df4b2 2251
39307a8e 2252 dirty_i->victim_secmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
5ec4e49f 2253 if (!dirty_i->victim_secmap)
351df4b2
JK		 2254 return -ENOMEM;
2255 return 0;
2256}
2257
2258static int build_dirty_segmap(struct f2fs_sb_info *sbi)
2259{
2260 struct dirty_seglist_info *dirty_i;
2261 unsigned int bitmap_size, i;
2262
2263 /* allocate memory for dirty segments list information */
2264 dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
2265 if (!dirty_i)
2266 return -ENOMEM;
2267
2268 SM_I(sbi)->dirty_info = dirty_i;
2269 mutex_init(&dirty_i->seglist_lock);
2270
7cd8558b 2271 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
351df4b2
JK		 2272
2273 for (i = 0; i < NR_DIRTY_TYPE; i++) {
39307a8e 2274 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
351df4b2
JK		 2275 if (!dirty_i->dirty_segmap[i])
2276 return -ENOMEM;
2277 }
2278
2279 init_dirty_segmap(sbi);
5ec4e49f 2280 return init_victim_secmap(sbi);
351df4b2
JK		 2281}
2282
0a8165d7 2283/*
351df4b2
JK		 2284 * Update min, max modified time for cost-benefit GC algorithm
2285 */
2286static void init_min_max_mtime(struct f2fs_sb_info *sbi)
2287{
2288 struct sit_info *sit_i = SIT_I(sbi);
2289 unsigned int segno;
2290
2291 mutex_lock(&sit_i->sentry_lock);
2292
2293 sit_i->min_mtime = LLONG_MAX;
2294
7cd8558b 2295 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
351df4b2
JK		 2296 unsigned int i;
2297 unsigned long long mtime = 0;
2298
2299 for (i = 0; i < sbi->segs_per_sec; i++)
2300 mtime += get_seg_entry(sbi, segno + i)->mtime;
2301
2302 mtime = div_u64(mtime, sbi->segs_per_sec);
2303
2304 if (sit_i->min_mtime > mtime)
2305 sit_i->min_mtime = mtime;
2306 }
2307 sit_i->max_mtime = get_mtime(sbi);
2308 mutex_unlock(&sit_i->sentry_lock);
2309}
2310
2311int build_segment_manager(struct f2fs_sb_info *sbi)
2312{
2313 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2314 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1042d60f 2315 struct f2fs_sm_info *sm_info;
351df4b2
JK		 2316 int err;
2317
2318 sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
2319 if (!sm_info)
2320 return -ENOMEM;
2321
2322 /* init sm info */
2323 sbi->sm_info = sm_info;
351df4b2
JK		 2324 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
2325 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
2326 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
2327 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
2328 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
2329 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
2330 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
58c41035
JK		 2331 sm_info->rec_prefree_segments = sm_info->main_segments *
2332 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
9b5f136f 2333 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
216fbd64 2334 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
c1ce1b02 2335 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
351df4b2 2336
7fd9e544
JK		 2337 INIT_LIST_HEAD(&sm_info->discard_list);
2338 sm_info->nr_discards = 0;
2339 sm_info->max_discards = 0;
2340
bba681cb
JK		 2341 sm_info->trim_sections = DEF_BATCHED_TRIM_SECTIONS;
2342
184a5cd2
CY		 2343 INIT_LIST_HEAD(&sm_info->sit_entry_set);
2344
b270ad6f 2345 if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) {
2163d198
GZ		 2346 err = create_flush_cmd_control(sbi);
2347 if (err)
a688b9d9 2348 return err;
6b4afdd7
JK		 2349 }
2350
351df4b2
JK		 2351 err = build_sit_info(sbi);
2352 if (err)
2353 return err;
2354 err = build_free_segmap(sbi);
2355 if (err)
2356 return err;
2357 err = build_curseg(sbi);
2358 if (err)
2359 return err;
2360
2361 /* reinit free segmap based on SIT */
2362 build_sit_entries(sbi);
2363
2364 init_free_segmap(sbi);
2365 err = build_dirty_segmap(sbi);
2366 if (err)
2367 return err;
2368
2369 init_min_max_mtime(sbi);
2370 return 0;
2371}
2372
2373static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
2374 enum dirty_type dirty_type)
2375{
2376 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2377
2378 mutex_lock(&dirty_i->seglist_lock);
39307a8e 2379 kvfree(dirty_i->dirty_segmap[dirty_type]);
351df4b2
JK		 2380 dirty_i->nr_dirty[dirty_type] = 0;
2381 mutex_unlock(&dirty_i->seglist_lock);
2382}
2383
5ec4e49f 2384static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
351df4b2
JK		 2385{
2386 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
39307a8e 2387 kvfree(dirty_i->victim_secmap);
351df4b2
JK		 2388}
2389
2390static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
2391{
2392 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2393 int i;
2394
2395 if (!dirty_i)
2396 return;
2397
2398 /* discard pre-free/dirty segments list */
2399 for (i = 0; i < NR_DIRTY_TYPE; i++)
2400 discard_dirty_segmap(sbi, i);
2401
5ec4e49f 2402 destroy_victim_secmap(sbi);
351df4b2
JK		 2403 SM_I(sbi)->dirty_info = NULL;
2404 kfree(dirty_i);
2405}
2406
2407static void destroy_curseg(struct f2fs_sb_info *sbi)
2408{
2409 struct curseg_info *array = SM_I(sbi)->curseg_array;
2410 int i;
2411
2412 if (!array)
2413 return;
2414 SM_I(sbi)->curseg_array = NULL;
2415 for (i = 0; i < NR_CURSEG_TYPE; i++)
2416 kfree(array[i].sum_blk);
2417 kfree(array);
2418}
2419
2420static void destroy_free_segmap(struct f2fs_sb_info *sbi)
2421{
2422 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
2423 if (!free_i)
2424 return;
2425 SM_I(sbi)->free_info = NULL;
39307a8e
JK		 2426 kvfree(free_i->free_segmap);
2427 kvfree(free_i->free_secmap);
351df4b2
JK		 2428 kfree(free_i);
2429}
2430
2431static void destroy_sit_info(struct f2fs_sb_info *sbi)
2432{
2433 struct sit_info *sit_i = SIT_I(sbi);
2434 unsigned int start;
2435
2436 if (!sit_i)
2437 return;
2438
2439 if (sit_i->sentries) {
7cd8558b 2440 for (start = 0; start < MAIN_SEGS(sbi); start++) {
351df4b2
JK		 2441 kfree(sit_i->sentries[start].cur_valid_map);
2442 kfree(sit_i->sentries[start].ckpt_valid_map);
a66cdd98 2443 kfree(sit_i->sentries[start].discard_map);
351df4b2
JK		 2444 }
2445 }
60a3b782
JK		 2446 kfree(sit_i->tmp_map);
2447
39307a8e
JK		 2448 kvfree(sit_i->sentries);
2449 kvfree(sit_i->sec_entries);
2450 kvfree(sit_i->dirty_sentries_bitmap);
351df4b2
JK		 2451
2452 SM_I(sbi)->sit_info = NULL;
2453 kfree(sit_i->sit_bitmap);
2454 kfree(sit_i);
2455}
2456
2457void destroy_segment_manager(struct f2fs_sb_info *sbi)
2458{
2459 struct f2fs_sm_info *sm_info = SM_I(sbi);
a688b9d9 2460
3b03f724
CY		 2461 if (!sm_info)
2462 return;
2163d198 2463 destroy_flush_cmd_control(sbi);
351df4b2
JK		 2464 destroy_dirty_segmap(sbi);
2465 destroy_curseg(sbi);
2466 destroy_free_segmap(sbi);
2467 destroy_sit_info(sbi);
2468 sbi->sm_info = NULL;
2469 kfree(sm_info);
2470}
7fd9e544
JK		 2471
2472int __init create_segment_manager_caches(void)
2473{
2474 discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
e8512d2e 2475 sizeof(struct discard_entry));
7fd9e544 2476 if (!discard_entry_slab)
184a5cd2
CY		 2477 goto fail;
2478
2479 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
c9ee0085 2480 sizeof(struct sit_entry_set));
184a5cd2
CY		 2481 if (!sit_entry_set_slab)
2482 goto destory_discard_entry;
88b88a66
JK		 2483
2484 inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry",
2485 sizeof(struct inmem_pages));
2486 if (!inmem_entry_slab)
2487 goto destroy_sit_entry_set;
7fd9e544 2488 return 0;
184a5cd2 2489
88b88a66
JK		 2490destroy_sit_entry_set:
2491 kmem_cache_destroy(sit_entry_set_slab);
184a5cd2
CY		 2492destory_discard_entry:
2493 kmem_cache_destroy(discard_entry_slab);
2494fail:
2495 return -ENOMEM;
7fd9e544
JK		 2496}
2497
2498void destroy_segment_manager_caches(void)
2499{
184a5cd2 2500 kmem_cache_destroy(sit_entry_set_slab);
7fd9e544 2501 kmem_cache_destroy(discard_entry_slab);
88b88a66 2502 kmem_cache_destroy(inmem_entry_slab);
7fd9e544 2503}
Linux kernel - Deliverables
This page took 0.33067 seconds and 5 git commands to generate.