fs/f2fs/segment.c

   1 /*
   2  * fs/f2fs/segment.c
   3  *
   4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   5  *             http://www.samsung.com/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/fs.h>
  12 #include <linux/f2fs_fs.h>
  13 #include <linux/bio.h>
  14 #include <linux/blkdev.h>
  15 #include <linux/prefetch.h>
  16 #include <linux/kthread.h>
  17 #include <linux/swap.h>
  18 #include <linux/timer.h>
  19
  20 #include "f2fs.h"
  21 #include "segment.h"
  22 #include "node.h"
  23 #include "trace.h"
  24 #include <trace/events/f2fs.h>
  25
  26 #define __reverse_ffz(x) __reverse_ffs(~(x))
  27
  28 static struct kmem_cache *discard_entry_slab;
  29 static struct kmem_cache *sit_entry_set_slab;
  30 static struct kmem_cache *inmem_entry_slab;
  31
  32 /*
  33  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
  34  * MSB and LSB are reversed in a byte by f2fs_set_bit.
  35  */
  36 static inline unsigned long __reverse_ffs(unsigned long word)
  37 {
  38         int num = 0;
  39
  40 #if BITS_PER_LONG == 64
  41         if ((word & 0xffffffff) == 0) {
  42                 num += 32;
  43                 word >>= 32;
  44         }
  45 #endif
  46         if ((word & 0xffff) == 0) {
  47                 num += 16;
  48                 word >>= 16;
  49         }
  50         if ((word & 0xff) == 0) {
  51                 num += 8;
  52                 word >>= 8;
  53         }
  54         if ((word & 0xf0) == 0)
  55                 num += 4;
  56         else
  57                 word >>= 4;
  58         if ((word & 0xc) == 0)
  59                 num += 2;
  60         else
  61                 word >>= 2;
  62         if ((word & 0x2) == 0)
  63                 num += 1;
  64         return num;
  65 }
  66
  67 /*
  68  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
  69  * f2fs_set_bit makes MSB and LSB reversed in a byte.
  70  * Example:
  71  *                             LSB <--> MSB
  72  *   f2fs_set_bit(0, bitmap) => 0000 0001
  73  *   f2fs_set_bit(7, bitmap) => 1000 0000
  74  */
  75 static unsigned long __find_rev_next_bit(const unsigned long *addr,
  76                         unsigned long size, unsigned long offset)
  77 {
  78         while (!f2fs_test_bit(offset, (unsigned char *)addr))
  79                 offset++;
  80
  81         if (offset > size)
  82                 offset = size;
  83
  84         return offset;
  85 #if 0
  86         const unsigned long *p = addr + BIT_WORD(offset);
  87         unsigned long result = offset & ~(BITS_PER_LONG - 1);
  88         unsigned long tmp;
  89         unsigned long mask, submask;
  90         unsigned long quot, rest;
  91
  92         if (offset >= size)
  93                 return size;
  94
  95         size -= result;
  96         offset %= BITS_PER_LONG;
  97         if (!offset)
  98                 goto aligned;
  99
 100         tmp = *(p++);
 101         quot = (offset >> 3) << 3;
 102         rest = offset & 0x7;
 103         mask = ~0UL << quot;
 104         submask = (unsigned char)(0xff << rest) >> rest;
 105         submask <<= quot;
 106         mask &= submask;
 107         tmp &= mask;
 108         if (size < BITS_PER_LONG)
 109                 goto found_first;
 110         if (tmp)
 111                 goto found_middle;
 112
 113         size -= BITS_PER_LONG;
 114         result += BITS_PER_LONG;
 115 aligned:
 116         while (size & ~(BITS_PER_LONG-1)) {
 117                 tmp = *(p++);
 118                 if (tmp)
 119                         goto found_middle;
 120                 result += BITS_PER_LONG;
 121                 size -= BITS_PER_LONG;
 122         }
 123         if (!size)
 124                 return result;
 125         tmp = *p;
 126 found_first:
 127         tmp &= (~0UL >> (BITS_PER_LONG - size));
 128         if (tmp == 0UL)         /* Are any bits set? */
 129                 return result + size;   /* Nope. */
 130 found_middle:
 131         return result + __reverse_ffs(tmp);
 132 #endif
 133 }
 134
 135 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
 136                         unsigned long size, unsigned long offset)
 137 {
 138         while (f2fs_test_bit(offset, (unsigned char *)addr))
 139                 offset++;
 140
 141         if (offset > size)
 142                 offset = size;
 143
 144         return offset;
 145 #if 0
 146         const unsigned long *p = addr + BIT_WORD(offset);
 147         unsigned long result = offset & ~(BITS_PER_LONG - 1);
 148         unsigned long tmp;
 149         unsigned long mask, submask;
 150         unsigned long quot, rest;
 151
 152         if (offset >= size)
 153                 return size;
 154
 155         size -= result;
 156         offset %= BITS_PER_LONG;
 157         if (!offset)
 158                 goto aligned;
 159
 160         tmp = *(p++);
 161         quot = (offset >> 3) << 3;
 162         rest = offset & 0x7;
 163         mask = ~(~0UL << quot);
 164         submask = (unsigned char)~((unsigned char)(0xff << rest) >> rest);
 165         submask <<= quot;
 166         mask += submask;
 167         tmp |= mask;
 168         if (size < BITS_PER_LONG)
 169                 goto found_first;
 170         if (~tmp)
 171                 goto found_middle;
 172
 173         size -= BITS_PER_LONG;
 174         result += BITS_PER_LONG;
 175 aligned:
 176         while (size & ~(BITS_PER_LONG - 1)) {
 177                 tmp = *(p++);
 178                 if (~tmp)
 179                         goto found_middle;
 180                 result += BITS_PER_LONG;
 181                 size -= BITS_PER_LONG;
 182         }
 183         if (!size)
 184                 return result;
 185         tmp = *p;
 186
 187 found_first:
 188         tmp |= ~0UL << size;
 189         if (tmp == ~0UL)        /* Are any bits zero? */
 190                 return result + size;   /* Nope. */
 191 found_middle:
 192         return result + __reverse_ffz(tmp);
 193 #endif
 194 }
 195
 196 void register_inmem_page(struct inode *inode, struct page *page)
 197 {
 198         struct f2fs_inode_info *fi = F2FS_I(inode);
 199         struct inmem_pages *new;
 200
 201         f2fs_trace_pid(page);
 202
 203         set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
 204         SetPagePrivate(page);
 205
 206         new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
 207
 208         /* add atomic page indices to the list */
 209         new->page = page;
 210         INIT_LIST_HEAD(&new->list);
 211
 212         /* increase reference count with clean state */
 213         mutex_lock(&fi->inmem_lock);
 214         get_page(page);
 215         list_add_tail(&new->list, &fi->inmem_pages);
 216         inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
 217         mutex_unlock(&fi->inmem_lock);
 218
 219         trace_f2fs_register_inmem_page(page, INMEM);
 220 }
 221
 222 int commit_inmem_pages(struct inode *inode, bool abort)
 223 {
 224         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 225         struct f2fs_inode_info *fi = F2FS_I(inode);
 226         struct inmem_pages *cur, *tmp;
 227         bool submit_bio = false;
 228         struct f2fs_io_info fio = {
 229                 .sbi = sbi,
 230                 .type = DATA,
 231                 .rw = WRITE_SYNC | REQ_PRIO,
 232                 .encrypted_page = NULL,
 233         };
 234         int err = 0;
 235
 236         /*
 237          * The abort is true only when f2fs_evict_inode is called.
 238          * Basically, the f2fs_evict_inode doesn't produce any data writes, so
 239          * that we don't need to call f2fs_balance_fs.
 240          * Otherwise, f2fs_gc in f2fs_balance_fs can wait forever until this
 241          * inode becomes free by iget_locked in f2fs_iget.
 242          */
 243         if (!abort) {
 244                 f2fs_balance_fs(sbi);
 245                 f2fs_lock_op(sbi);
 246         }
 247
 248         mutex_lock(&fi->inmem_lock);
 249         list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
 250                 lock_page(cur->page);
 251                 if (!abort) {
 252                         if (cur->page->mapping == inode->i_mapping) {
 253                                 set_page_dirty(cur->page);
 254                                 f2fs_wait_on_page_writeback(cur->page, DATA);
 255                                 if (clear_page_dirty_for_io(cur->page))
 256                                         inode_dec_dirty_pages(inode);
 257                                 trace_f2fs_commit_inmem_page(cur->page, INMEM);
 258                                 fio.page = cur->page;
 259                                 err = do_write_data_page(&fio);
 260                                 submit_bio = true;
 261                                 if (err) {
 262                                         unlock_page(cur->page);
 263                                         break;
 264                                 }
 265                         }
 266                 } else {
 267                         trace_f2fs_commit_inmem_page(cur->page, INMEM_DROP);
 268                 }
 269                 set_page_private(cur->page, 0);
 270                 ClearPagePrivate(cur->page);
 271                 f2fs_put_page(cur->page, 1);
 272
 273                 list_del(&cur->list);
 274                 kmem_cache_free(inmem_entry_slab, cur);
 275                 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
 276         }
 277         mutex_unlock(&fi->inmem_lock);
 278
 279         if (!abort) {
 280                 f2fs_unlock_op(sbi);
 281                 if (submit_bio)
 282                         f2fs_submit_merged_bio(sbi, DATA, WRITE);
 283         }
 284         return err;
 285 }
 286
 287 /*
 288  * This function balances dirty node and dentry pages.
 289  * In addition, it controls garbage collection.
 290  */
 291 void f2fs_balance_fs(struct f2fs_sb_info *sbi)
 292 {
 293         /*
 294          * We should do GC or end up with checkpoint, if there are so many dirty
 295          * dir/node pages without enough free segments.
 296          */
 297         if (has_not_enough_free_secs(sbi, 0)) {
 298                 mutex_lock(&sbi->gc_mutex);
 299                 f2fs_gc(sbi, false);
 300         }
 301 }
 302
 303 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
 304 {
 305         /* try to shrink extent cache when there is no enough memory */
 306         if (!available_free_memory(sbi, EXTENT_CACHE))
 307                 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
 308
 309         /* check the # of cached NAT entries */
 310         if (!available_free_memory(sbi, NAT_ENTRIES))
 311                 try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
 312
 313         if (!available_free_memory(sbi, FREE_NIDS))
 314                 try_to_free_nids(sbi, NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES);
 315
 316         /* checkpoint is the only way to shrink partial cached entries */
 317         if (!available_free_memory(sbi, NAT_ENTRIES) ||
 318                         excess_prefree_segs(sbi) ||
 319                         !available_free_memory(sbi, INO_ENTRIES) ||
 320                         jiffies > sbi->cp_expires)
 321                 f2fs_sync_fs(sbi->sb, true);
 322 }
 323
 324 static int issue_flush_thread(void *data)
 325 {
 326         struct f2fs_sb_info *sbi = data;
 327         struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
 328         wait_queue_head_t *q = &fcc->flush_wait_queue;
 329 repeat:
 330         if (kthread_should_stop())
 331                 return 0;
 332
 333         if (!llist_empty(&fcc->issue_list)) {
 334                 struct bio *bio;
 335                 struct flush_cmd *cmd, *next;
 336                 int ret;
 337
 338                 bio = f2fs_bio_alloc(0);
 339
 340                 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
 341                 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
 342
 343                 bio->bi_bdev = sbi->sb->s_bdev;
 344                 ret = submit_bio_wait(WRITE_FLUSH, bio);
 345
 346                 llist_for_each_entry_safe(cmd, next,
 347                                           fcc->dispatch_list, llnode) {
 348                         cmd->ret = ret;
 349                         complete(&cmd->wait);
 350                 }
 351                 bio_put(bio);
 352                 fcc->dispatch_list = NULL;
 353         }
 354
 355         wait_event_interruptible(*q,
 356                 kthread_should_stop() || !llist_empty(&fcc->issue_list));
 357         goto repeat;
 358 }
 359
 360 int f2fs_issue_flush(struct f2fs_sb_info *sbi)
 361 {
 362         struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
 363         struct flush_cmd cmd;
 364
 365         trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER),
 366                                         test_opt(sbi, FLUSH_MERGE));
 367
 368         if (test_opt(sbi, NOBARRIER))
 369                 return 0;
 370
 371         if (!test_opt(sbi, FLUSH_MERGE)) {
 372                 struct bio *bio = f2fs_bio_alloc(0);
 373                 int ret;
 374
 375                 bio->bi_bdev = sbi->sb->s_bdev;
 376                 ret = submit_bio_wait(WRITE_FLUSH, bio);
 377                 bio_put(bio);
 378                 return ret;
 379         }
 380
 381         init_completion(&cmd.wait);
 382
 383         llist_add(&cmd.llnode, &fcc->issue_list);
 384
 385         if (!fcc->dispatch_list)
 386                 wake_up(&fcc->flush_wait_queue);
 387
 388         wait_for_completion(&cmd.wait);
 389
 390         return cmd.ret;
 391 }
 392
 393 int create_flush_cmd_control(struct f2fs_sb_info *sbi)
 394 {
 395         dev_t dev = sbi->sb->s_bdev->bd_dev;
 396         struct flush_cmd_control *fcc;
 397         int err = 0;
 398
 399         fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL);
 400         if (!fcc)
 401                 return -ENOMEM;
 402         init_waitqueue_head(&fcc->flush_wait_queue);
 403         init_llist_head(&fcc->issue_list);
 404         SM_I(sbi)->cmd_control_info = fcc;
 405         fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
 406                                 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
 407         if (IS_ERR(fcc->f2fs_issue_flush)) {
 408                 err = PTR_ERR(fcc->f2fs_issue_flush);
 409                 kfree(fcc);
 410                 SM_I(sbi)->cmd_control_info = NULL;
 411                 return err;
 412         }
 413
 414         return err;
 415 }
 416
 417 void destroy_flush_cmd_control(struct f2fs_sb_info *sbi)
 418 {
 419         struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
 420
 421         if (fcc && fcc->f2fs_issue_flush)
 422                 kthread_stop(fcc->f2fs_issue_flush);
 423         kfree(fcc);
 424         SM_I(sbi)->cmd_control_info = NULL;
 425 }
 426
 427 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 428                 enum dirty_type dirty_type)
 429 {
 430         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 431
 432         /* need not be added */
 433         if (IS_CURSEG(sbi, segno))
 434                 return;
 435
 436         if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 437                 dirty_i->nr_dirty[dirty_type]++;
 438
 439         if (dirty_type == DIRTY) {
 440                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 441                 enum dirty_type t = sentry->type;
 442
 443                 if (unlikely(t >= DIRTY)) {
 444                         f2fs_bug_on(sbi, 1);
 445                         return;
 446                 }
 447                 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
 448                         dirty_i->nr_dirty[t]++;
 449         }
 450 }
 451
 452 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 453                 enum dirty_type dirty_type)
 454 {
 455         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 456
 457         if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 458                 dirty_i->nr_dirty[dirty_type]--;
 459
 460         if (dirty_type == DIRTY) {
 461                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 462                 enum dirty_type t = sentry->type;
 463
 464                 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
 465                         dirty_i->nr_dirty[t]--;
 466
 467                 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
 468                         clear_bit(GET_SECNO(sbi, segno),
 469                                                 dirty_i->victim_secmap);
 470         }
 471 }
 472
 473 /*
 474  * Should not occur error such as -ENOMEM.
 475  * Adding dirty entry into seglist is not critical operation.
 476  * If a given segment is one of current working segments, it won't be added.
 477  */
 478 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
 479 {
 480         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 481         unsigned short valid_blocks;
 482
 483         if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
 484                 return;
 485
 486         mutex_lock(&dirty_i->seglist_lock);
 487
 488         valid_blocks = get_valid_blocks(sbi, segno, 0);
 489
 490         if (valid_blocks == 0) {
 491                 __locate_dirty_segment(sbi, segno, PRE);
 492                 __remove_dirty_segment(sbi, segno, DIRTY);
 493         } else if (valid_blocks < sbi->blocks_per_seg) {
 494                 __locate_dirty_segment(sbi, segno, DIRTY);
 495         } else {
 496                 /* Recovery routine with SSR needs this */
 497                 __remove_dirty_segment(sbi, segno, DIRTY);
 498         }
 499
 500         mutex_unlock(&dirty_i->seglist_lock);
 501 }
 502
 503 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
 504                                 block_t blkstart, block_t blklen)
 505 {
 506         sector_t start = SECTOR_FROM_BLOCK(blkstart);
 507         sector_t len = SECTOR_FROM_BLOCK(blklen);
 508         struct seg_entry *se;
 509         unsigned int offset;
 510         block_t i;
 511
 512         for (i = blkstart; i < blkstart + blklen; i++) {
 513                 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
 514                 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
 515
 516                 if (!f2fs_test_and_set_bit(offset, se->discard_map))
 517                         sbi->discard_blks--;
 518         }
 519         trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
 520         return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0);
 521 }
 522
 523 bool discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr)
 524 {
 525         int err = -ENOTSUPP;
 526
 527         if (test_opt(sbi, DISCARD)) {
 528                 struct seg_entry *se = get_seg_entry(sbi,
 529                                 GET_SEGNO(sbi, blkaddr));
 530                 unsigned int offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 531
 532                 if (f2fs_test_bit(offset, se->discard_map))
 533                         return false;
 534
 535                 err = f2fs_issue_discard(sbi, blkaddr, 1);
 536         }
 537
 538         if (err) {
 539                 update_meta_page(sbi, NULL, blkaddr);
 540                 return true;
 541         }
 542         return false;
 543 }
 544
 545 static void __add_discard_entry(struct f2fs_sb_info *sbi,
 546                 struct cp_control *cpc, struct seg_entry *se,
 547                 unsigned int start, unsigned int end)
 548 {
 549         struct list_head *head = &SM_I(sbi)->discard_list;
 550         struct discard_entry *new, *last;
 551
 552         if (!list_empty(head)) {
 553                 last = list_last_entry(head, struct discard_entry, list);
 554                 if (START_BLOCK(sbi, cpc->trim_start) + start ==
 555                                                 last->blkaddr + last->len) {
 556                         last->len += end - start;
 557                         goto done;
 558                 }
 559         }
 560
 561         new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
 562         INIT_LIST_HEAD(&new->list);
 563         new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start;
 564         new->len = end - start;
 565         list_add_tail(&new->list, head);
 566 done:
 567         SM_I(sbi)->nr_discards += end - start;
 568 }
 569
 570 static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 571 {
 572         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
 573         int max_blocks = sbi->blocks_per_seg;
 574         struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
 575         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
 576         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
 577         unsigned long *discard_map = (unsigned long *)se->discard_map;
 578         unsigned long *dmap = SIT_I(sbi)->tmp_map;
 579         unsigned int start = 0, end = -1;
 580         bool force = (cpc->reason == CP_DISCARD);
 581         int i;
 582
 583         if (se->valid_blocks == max_blocks)
 584                 return;
 585
 586         if (!force) {
 587                 if (!test_opt(sbi, DISCARD) || !se->valid_blocks ||
 588                     SM_I(sbi)->nr_discards >= SM_I(sbi)->max_discards)
 589                         return;
 590         }
 591
 592         /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
 593         for (i = 0; i < entries; i++)
 594                 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
 595                                 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
 596
 597         while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
 598                 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
 599                 if (start >= max_blocks)
 600                         break;
 601
 602                 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
 603                 __add_discard_entry(sbi, cpc, se, start, end);
 604         }
 605 }
 606
 607 void release_discard_addrs(struct f2fs_sb_info *sbi)
 608 {
 609         struct list_head *head = &(SM_I(sbi)->discard_list);
 610         struct discard_entry *entry, *this;
 611
 612         /* drop caches */
 613         list_for_each_entry_safe(entry, this, head, list) {
 614                 list_del(&entry->list);
 615                 kmem_cache_free(discard_entry_slab, entry);
 616         }
 617 }
 618
 619 /*
 620  * Should call clear_prefree_segments after checkpoint is done.
 621  */
 622 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
 623 {
 624         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 625         unsigned int segno;
 626
 627         mutex_lock(&dirty_i->seglist_lock);
 628         for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
 629                 __set_test_and_free(sbi, segno);
 630         mutex_unlock(&dirty_i->seglist_lock);
 631 }
 632
 633 void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 634 {
 635         struct list_head *head = &(SM_I(sbi)->discard_list);
 636         struct discard_entry *entry, *this;
 637         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 638         unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
 639         unsigned int start = 0, end = -1;
 640
 641         mutex_lock(&dirty_i->seglist_lock);
 642
 643         while (1) {
 644                 int i;
 645                 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
 646                 if (start >= MAIN_SEGS(sbi))
 647                         break;
 648                 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
 649                                                                 start + 1);
 650
 651                 for (i = start; i < end; i++)
 652                         clear_bit(i, prefree_map);
 653
 654                 dirty_i->nr_dirty[PRE] -= end - start;
 655
 656                 if (!test_opt(sbi, DISCARD))
 657                         continue;
 658
 659                 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
 660                                 (end - start) << sbi->log_blocks_per_seg);
 661         }
 662         mutex_unlock(&dirty_i->seglist_lock);
 663
 664         /* send small discards */
 665         list_for_each_entry_safe(entry, this, head, list) {
 666                 if (cpc->reason == CP_DISCARD && entry->len < cpc->trim_minlen)
 667                         goto skip;
 668                 f2fs_issue_discard(sbi, entry->blkaddr, entry->len);
 669                 cpc->trimmed += entry->len;
 670 skip:
 671                 list_del(&entry->list);
 672                 SM_I(sbi)->nr_discards -= entry->len;
 673                 kmem_cache_free(discard_entry_slab, entry);
 674         }
 675 }
 676
 677 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
 678 {
 679         struct sit_info *sit_i = SIT_I(sbi);
 680
 681         if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
 682                 sit_i->dirty_sentries++;
 683                 return false;
 684         }
 685
 686         return true;
 687 }
 688
 689 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
 690                                         unsigned int segno, int modified)
 691 {
 692         struct seg_entry *se = get_seg_entry(sbi, segno);
 693         se->type = type;
 694         if (modified)
 695                 __mark_sit_entry_dirty(sbi, segno);
 696 }
 697
 698 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
 699 {
 700         struct seg_entry *se;
 701         unsigned int segno, offset;
 702         long int new_vblocks;
 703
 704         segno = GET_SEGNO(sbi, blkaddr);
 705
 706         se = get_seg_entry(sbi, segno);
 707         new_vblocks = se->valid_blocks + del;
 708         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 709
 710         f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) ||
 711                                 (new_vblocks > sbi->blocks_per_seg)));
 712
 713         se->valid_blocks = new_vblocks;
 714         se->mtime = get_mtime(sbi);
 715         SIT_I(sbi)->max_mtime = se->mtime;
 716
 717         /* Update valid block bitmap */
 718         if (del > 0) {
 719                 if (f2fs_test_and_set_bit(offset, se->cur_valid_map))
 720                         f2fs_bug_on(sbi, 1);
 721                 if (!f2fs_test_and_set_bit(offset, se->discard_map))
 722                         sbi->discard_blks--;
 723         } else {
 724                 if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map))
 725                         f2fs_bug_on(sbi, 1);
 726                 if (f2fs_test_and_clear_bit(offset, se->discard_map))
 727                         sbi->discard_blks++;
 728         }
 729         if (!f2fs_test_bit(offset, se->ckpt_valid_map))
 730                 se->ckpt_valid_blocks += del;
 731
 732         __mark_sit_entry_dirty(sbi, segno);
 733
 734         /* update total number of valid blocks to be written in ckpt area */
 735         SIT_I(sbi)->written_valid_blocks += del;
 736
 737         if (sbi->segs_per_sec > 1)
 738                 get_sec_entry(sbi, segno)->valid_blocks += del;
 739 }
 740
 741 void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new)
 742 {
 743         update_sit_entry(sbi, new, 1);
 744         if (GET_SEGNO(sbi, old) != NULL_SEGNO)
 745                 update_sit_entry(sbi, old, -1);
 746
 747         locate_dirty_segment(sbi, GET_SEGNO(sbi, old));
 748         locate_dirty_segment(sbi, GET_SEGNO(sbi, new));
 749 }
 750
 751 void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
 752 {
 753         unsigned int segno = GET_SEGNO(sbi, addr);
 754         struct sit_info *sit_i = SIT_I(sbi);
 755
 756         f2fs_bug_on(sbi, addr == NULL_ADDR);
 757         if (addr == NEW_ADDR)
 758                 return;
 759
 760         /* add it into sit main buffer */
 761         mutex_lock(&sit_i->sentry_lock);
 762
 763         update_sit_entry(sbi, addr, -1);
 764
 765         /* add it into dirty seglist */
 766         locate_dirty_segment(sbi, segno);
 767
 768         mutex_unlock(&sit_i->sentry_lock);
 769 }
 770
 771 bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
 772 {
 773         struct sit_info *sit_i = SIT_I(sbi);
 774         unsigned int segno, offset;
 775         struct seg_entry *se;
 776         bool is_cp = false;
 777
 778         if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR)
 779                 return true;
 780
 781         mutex_lock(&sit_i->sentry_lock);
 782
 783         segno = GET_SEGNO(sbi, blkaddr);
 784         se = get_seg_entry(sbi, segno);
 785         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 786
 787         if (f2fs_test_bit(offset, se->ckpt_valid_map))
 788                 is_cp = true;
 789
 790         mutex_unlock(&sit_i->sentry_lock);
 791
 792         return is_cp;
 793 }
 794
 795 /*
 796  * This function should be resided under the curseg_mutex lock
 797  */
 798 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
 799                                         struct f2fs_summary *sum)
 800 {
 801         struct curseg_info *curseg = CURSEG_I(sbi, type);
 802         void *addr = curseg->sum_blk;
 803         addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
 804         memcpy(addr, sum, sizeof(struct f2fs_summary));
 805 }
 806
 807 /*
 808  * Calculate the number of current summary pages for writing
 809  */
 810 int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
 811 {
 812         int valid_sum_count = 0;
 813         int i, sum_in_page;
 814
 815         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
 816                 if (sbi->ckpt->alloc_type[i] == SSR)
 817                         valid_sum_count += sbi->blocks_per_seg;
 818                 else {
 819                         if (for_ra)
 820                                 valid_sum_count += le16_to_cpu(
 821                                         F2FS_CKPT(sbi)->cur_data_blkoff[i]);
 822                         else
 823                                 valid_sum_count += curseg_blkoff(sbi, i);
 824                 }
 825         }
 826
 827         sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE -
 828                         SUM_FOOTER_SIZE) / SUMMARY_SIZE;
 829         if (valid_sum_count <= sum_in_page)
 830                 return 1;
 831         else if ((valid_sum_count - sum_in_page) <=
 832                 (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
 833                 return 2;
 834         return 3;
 835 }
 836
 837 /*
 838  * Caller should put this summary page
 839  */
 840 struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
 841 {
 842         return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
 843 }
 844
 845 void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr)
 846 {
 847         struct page *page = grab_meta_page(sbi, blk_addr);
 848         void *dst = page_address(page);
 849
 850         if (src)
 851                 memcpy(dst, src, PAGE_CACHE_SIZE);
 852         else
 853                 memset(dst, 0, PAGE_CACHE_SIZE);
 854         set_page_dirty(page);
 855         f2fs_put_page(page, 1);
 856 }
 857
 858 static void write_sum_page(struct f2fs_sb_info *sbi,
 859                         struct f2fs_summary_block *sum_blk, block_t blk_addr)
 860 {
 861         update_meta_page(sbi, (void *)sum_blk, blk_addr);
 862 }
 863
 864 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
 865 {
 866         struct curseg_info *curseg = CURSEG_I(sbi, type);
 867         unsigned int segno = curseg->segno + 1;
 868         struct free_segmap_info *free_i = FREE_I(sbi);
 869
 870         if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
 871                 return !test_bit(segno, free_i->free_segmap);
 872         return 0;
 873 }
 874
 875 /*
 876  * Find a new segment from the free segments bitmap to right order
 877  * This function should be returned with success, otherwise BUG
 878  */
 879 static void get_new_segment(struct f2fs_sb_info *sbi,
 880                         unsigned int *newseg, bool new_sec, int dir)
 881 {
 882         struct free_segmap_info *free_i = FREE_I(sbi);
 883         unsigned int segno, secno, zoneno;
 884         unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
 885         unsigned int hint = *newseg / sbi->segs_per_sec;
 886         unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
 887         unsigned int left_start = hint;
 888         bool init = true;
 889         int go_left = 0;
 890         int i;
 891
 892         spin_lock(&free_i->segmap_lock);
 893
 894         if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
 895                 segno = find_next_zero_bit(free_i->free_segmap,
 896                                         MAIN_SEGS(sbi), *newseg + 1);
 897                 if (segno - *newseg < sbi->segs_per_sec -
 898                                         (*newseg % sbi->segs_per_sec))
 899                         goto got_it;
 900         }
 901 find_other_zone:
 902         secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
 903         if (secno >= MAIN_SECS(sbi)) {
 904                 if (dir == ALLOC_RIGHT) {
 905                         secno = find_next_zero_bit(free_i->free_secmap,
 906                                                         MAIN_SECS(sbi), 0);
 907                         f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
 908                 } else {
 909                         go_left = 1;
 910                         left_start = hint - 1;
 911                 }
 912         }
 913         if (go_left == 0)
 914                 goto skip_left;
 915
 916         while (test_bit(left_start, free_i->free_secmap)) {
 917                 if (left_start > 0) {
 918                         left_start--;
 919                         continue;
 920                 }
 921                 left_start = find_next_zero_bit(free_i->free_secmap,
 922                                                         MAIN_SECS(sbi), 0);
 923                 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
 924                 break;
 925         }
 926         secno = left_start;
 927 skip_left:
 928         hint = secno;
 929         segno = secno * sbi->segs_per_sec;
 930         zoneno = secno / sbi->secs_per_zone;
 931
 932         /* give up on finding another zone */
 933         if (!init)
 934                 goto got_it;
 935         if (sbi->secs_per_zone == 1)
 936                 goto got_it;
 937         if (zoneno == old_zoneno)
 938                 goto got_it;
 939         if (dir == ALLOC_LEFT) {
 940                 if (!go_left && zoneno + 1 >= total_zones)
 941                         goto got_it;
 942                 if (go_left && zoneno == 0)
 943                         goto got_it;
 944         }
 945         for (i = 0; i < NR_CURSEG_TYPE; i++)
 946                 if (CURSEG_I(sbi, i)->zone == zoneno)
 947                         break;
 948
 949         if (i < NR_CURSEG_TYPE) {
 950                 /* zone is in user, try another */
 951                 if (go_left)
 952                         hint = zoneno * sbi->secs_per_zone - 1;
 953                 else if (zoneno + 1 >= total_zones)
 954                         hint = 0;
 955                 else
 956                         hint = (zoneno + 1) * sbi->secs_per_zone;
 957                 init = false;
 958                 goto find_other_zone;
 959         }
 960 got_it:
 961         /* set it as dirty segment in free segmap */
 962         f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
 963         __set_inuse(sbi, segno);
 964         *newseg = segno;
 965         spin_unlock(&free_i->segmap_lock);
 966 }
 967
 968 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
 969 {
 970         struct curseg_info *curseg = CURSEG_I(sbi, type);
 971         struct summary_footer *sum_footer;
 972
 973         curseg->segno = curseg->next_segno;
 974         curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
 975         curseg->next_blkoff = 0;
 976         curseg->next_segno = NULL_SEGNO;
 977
 978         sum_footer = &(curseg->sum_blk->footer);
 979         memset(sum_footer, 0, sizeof(struct summary_footer));
 980         if (IS_DATASEG(type))
 981                 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
 982         if (IS_NODESEG(type))
 983                 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
 984         __set_sit_entry_type(sbi, type, curseg->segno, modified);
 985 }
 986
 987 /*
 988  * Allocate a current working segment.
 989  * This function always allocates a free segment in LFS manner.
 990  */
 991 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
 992 {
 993         struct curseg_info *curseg = CURSEG_I(sbi, type);
 994         unsigned int segno = curseg->segno;
 995         int dir = ALLOC_LEFT;
 996
 997         write_sum_page(sbi, curseg->sum_blk,
 998                                 GET_SUM_BLOCK(sbi, segno));
 999         if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
1000                 dir = ALLOC_RIGHT;
1001
1002         if (test_opt(sbi, NOHEAP))
1003                 dir = ALLOC_RIGHT;
1004
1005         get_new_segment(sbi, &segno, new_sec, dir);
1006         curseg->next_segno = segno;
1007         reset_curseg(sbi, type, 1);
1008         curseg->alloc_type = LFS;
1009 }
1010
1011 static void __next_free_blkoff(struct f2fs_sb_info *sbi,
1012                         struct curseg_info *seg, block_t start)
1013 {
1014         struct seg_entry *se = get_seg_entry(sbi, seg->segno);
1015         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1016         unsigned long *target_map = SIT_I(sbi)->tmp_map;
1017         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1018         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1019         int i, pos;
1020
1021         for (i = 0; i < entries; i++)
1022                 target_map[i] = ckpt_map[i] | cur_map[i];
1023
1024         pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
1025
1026         seg->next_blkoff = pos;
1027 }
1028
1029 /*
1030  * If a segment is written by LFS manner, next block offset is just obtained
1031  * by increasing the current block offset. However, if a segment is written by
1032  * SSR manner, next block offset obtained by calling __next_free_blkoff
1033  */
1034 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
1035                                 struct curseg_info *seg)
1036 {
1037         if (seg->alloc_type == SSR)
1038                 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
1039         else
1040                 seg->next_blkoff++;
1041 }
1042
1043 /*
1044  * This function always allocates a used segment(from dirty seglist) by SSR
1045  * manner, so it should recover the existing segment information of valid blocks
1046  */
1047 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
1048 {
1049         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1050         struct curseg_info *curseg = CURSEG_I(sbi, type);
1051         unsigned int new_segno = curseg->next_segno;
1052         struct f2fs_summary_block *sum_node;
1053         struct page *sum_page;
1054
1055         write_sum_page(sbi, curseg->sum_blk,
1056                                 GET_SUM_BLOCK(sbi, curseg->segno));
1057         __set_test_and_inuse(sbi, new_segno);
1058
1059         mutex_lock(&dirty_i->seglist_lock);
1060         __remove_dirty_segment(sbi, new_segno, PRE);
1061         __remove_dirty_segment(sbi, new_segno, DIRTY);
1062         mutex_unlock(&dirty_i->seglist_lock);
1063
1064         reset_curseg(sbi, type, 1);
1065         curseg->alloc_type = SSR;
1066         __next_free_blkoff(sbi, curseg, 0);
1067
1068         if (reuse) {
1069                 sum_page = get_sum_page(sbi, new_segno);
1070                 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
1071                 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
1072                 f2fs_put_page(sum_page, 1);
1073         }
1074 }
1075
1076 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
1077 {
1078         struct curseg_info *curseg = CURSEG_I(sbi, type);
1079         const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
1080
1081         if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0))
1082                 return v_ops->get_victim(sbi,
1083                                 &(curseg)->next_segno, BG_GC, type, SSR);
1084
1085         /* For data segments, let's do SSR more intensively */
1086         for (; type >= CURSEG_HOT_DATA; type--)
1087                 if (v_ops->get_victim(sbi, &(curseg)->next_segno,
1088                                                 BG_GC, type, SSR))
1089                         return 1;
1090         return 0;
1091 }
1092
1093 /*
1094  * flush out current segment and replace it with new segment
1095  * This function should be returned with success, otherwise BUG
1096  */
1097 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
1098                                                 int type, bool force)
1099 {
1100         struct curseg_info *curseg = CURSEG_I(sbi, type);
1101
1102         if (force)
1103                 new_curseg(sbi, type, true);
1104         else if (type == CURSEG_WARM_NODE)
1105                 new_curseg(sbi, type, false);
1106         else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
1107                 new_curseg(sbi, type, false);
1108         else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
1109                 change_curseg(sbi, type, true);
1110         else
1111                 new_curseg(sbi, type, false);
1112
1113         stat_inc_seg_type(sbi, curseg);
1114 }
1115
1116 static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type)
1117 {
1118         struct curseg_info *curseg = CURSEG_I(sbi, type);
1119         unsigned int old_segno;
1120
1121         old_segno = curseg->segno;
1122         SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
1123         locate_dirty_segment(sbi, old_segno);
1124 }
1125
1126 void allocate_new_segments(struct f2fs_sb_info *sbi)
1127 {
1128         int i;
1129
1130         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
1131                 __allocate_new_segments(sbi, i);
1132 }
1133
1134 static const struct segment_allocation default_salloc_ops = {
1135         .allocate_segment = allocate_segment_by_default,
1136 };
1137
1138 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
1139 {
1140         __u64 start = F2FS_BYTES_TO_BLK(range->start);
1141         __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
1142         unsigned int start_segno, end_segno;
1143         struct cp_control cpc;
1144
1145         if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
1146                 return -EINVAL;
1147
1148         cpc.trimmed = 0;
1149         if (end <= MAIN_BLKADDR(sbi))
1150                 goto out;
1151
1152         /* start/end segment number in main_area */
1153         start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
1154         end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
1155                                                 GET_SEGNO(sbi, end);
1156         cpc.reason = CP_DISCARD;
1157         cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
1158
1159         /* do checkpoint to issue discard commands safely */
1160         for (; start_segno <= end_segno; start_segno = cpc.trim_end + 1) {
1161                 cpc.trim_start = start_segno;
1162
1163                 if (sbi->discard_blks == 0)
1164                         break;
1165                 else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi))
1166                         cpc.trim_end = end_segno;
1167                 else
1168                         cpc.trim_end = min_t(unsigned int,
1169                                 rounddown(start_segno +
1170                                 BATCHED_TRIM_SEGMENTS(sbi),
1171                                 sbi->segs_per_sec) - 1, end_segno);
1172
1173                 mutex_lock(&sbi->gc_mutex);
1174                 write_checkpoint(sbi, &cpc);
1175                 mutex_unlock(&sbi->gc_mutex);
1176         }
1177 out:
1178         range->len = F2FS_BLK_TO_BYTES(cpc.trimmed);
1179         return 0;
1180 }
1181
1182 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
1183 {
1184         struct curseg_info *curseg = CURSEG_I(sbi, type);
1185         if (curseg->next_blkoff < sbi->blocks_per_seg)
1186                 return true;
1187         return false;
1188 }
1189
1190 static int __get_segment_type_2(struct page *page, enum page_type p_type)
1191 {
1192         if (p_type == DATA)
1193                 return CURSEG_HOT_DATA;
1194         else
1195                 return CURSEG_HOT_NODE;
1196 }
1197
1198 static int __get_segment_type_4(struct page *page, enum page_type p_type)
1199 {
1200         if (p_type == DATA) {
1201                 struct inode *inode = page->mapping->host;
1202
1203                 if (S_ISDIR(inode->i_mode))
1204                         return CURSEG_HOT_DATA;
1205                 else
1206                         return CURSEG_COLD_DATA;
1207         } else {
1208                 if (IS_DNODE(page) && is_cold_node(page))
1209                         return CURSEG_WARM_NODE;
1210                 else
1211                         return CURSEG_COLD_NODE;
1212         }
1213 }
1214
1215 static int __get_segment_type_6(struct page *page, enum page_type p_type)
1216 {
1217         if (p_type == DATA) {
1218                 struct inode *inode = page->mapping->host;
1219
1220                 if (S_ISDIR(inode->i_mode))
1221                         return CURSEG_HOT_DATA;
1222                 else if (is_cold_data(page) || file_is_cold(inode))
1223                         return CURSEG_COLD_DATA;
1224                 else
1225                         return CURSEG_WARM_DATA;
1226         } else {
1227                 if (IS_DNODE(page))
1228                         return is_cold_node(page) ? CURSEG_WARM_NODE :
1229                                                 CURSEG_HOT_NODE;
1230                 else
1231                         return CURSEG_COLD_NODE;
1232         }
1233 }
1234
1235 static int __get_segment_type(struct page *page, enum page_type p_type)
1236 {
1237         switch (F2FS_P_SB(page)->active_logs) {
1238         case 2:
1239                 return __get_segment_type_2(page, p_type);
1240         case 4:
1241                 return __get_segment_type_4(page, p_type);
1242         }
1243         /* NR_CURSEG_TYPE(6) logs by default */
1244         f2fs_bug_on(F2FS_P_SB(page),
1245                 F2FS_P_SB(page)->active_logs != NR_CURSEG_TYPE);
1246         return __get_segment_type_6(page, p_type);
1247 }
1248
1249 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
1250                 block_t old_blkaddr, block_t *new_blkaddr,
1251                 struct f2fs_summary *sum, int type)
1252 {
1253         struct sit_info *sit_i = SIT_I(sbi);
1254         struct curseg_info *curseg;
1255         bool direct_io = (type == CURSEG_DIRECT_IO);
1256
1257         type = direct_io ? CURSEG_WARM_DATA : type;
1258
1259         curseg = CURSEG_I(sbi, type);
1260
1261         mutex_lock(&curseg->curseg_mutex);
1262         mutex_lock(&sit_i->sentry_lock);
1263
1264         /* direct_io'ed data is aligned to the segment for better performance */
1265         if (direct_io && curseg->next_blkoff &&
1266                                 !has_not_enough_free_secs(sbi, 0))
1267                 __allocate_new_segments(sbi, type);
1268
1269         *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
1270
1271         /*
1272          * __add_sum_entry should be resided under the curseg_mutex
1273          * because, this function updates a summary entry in the
1274          * current summary block.
1275          */
1276         __add_sum_entry(sbi, type, sum);
1277
1278         __refresh_next_blkoff(sbi, curseg);
1279
1280         stat_inc_block_count(sbi, curseg);
1281
1282         if (!__has_curseg_space(sbi, type))
1283                 sit_i->s_ops->allocate_segment(sbi, type, false);
1284         /*
1285          * SIT information should be updated before segment allocation,
1286          * since SSR needs latest valid block information.
1287          */
1288         refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
1289
1290         mutex_unlock(&sit_i->sentry_lock);
1291
1292         if (page && IS_NODESEG(type))
1293                 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
1294
1295         mutex_unlock(&curseg->curseg_mutex);
1296 }
1297
1298 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
1299 {
1300         int type = __get_segment_type(fio->page, fio->type);
1301
1302         allocate_data_block(fio->sbi, fio->page, fio->blk_addr,
1303                                         &fio->blk_addr, sum, type);
1304
1305         /* writeout dirty page into bdev */
1306         f2fs_submit_page_mbio(fio);
1307 }
1308
1309 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
1310 {
1311         struct f2fs_io_info fio = {
1312                 .sbi = sbi,
1313                 .type = META,
1314                 .rw = WRITE_SYNC | REQ_META | REQ_PRIO,
1315                 .blk_addr = page->index,
1316                 .page = page,
1317                 .encrypted_page = NULL,
1318         };
1319
1320         if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
1321                 fio.rw &= ~REQ_META;
1322
1323         set_page_writeback(page);
1324         f2fs_submit_page_mbio(&fio);
1325 }
1326
1327 void write_node_page(unsigned int nid, struct f2fs_io_info *fio)
1328 {
1329         struct f2fs_summary sum;
1330
1331         set_summary(&sum, nid, 0, 0);
1332         do_write_page(&sum, fio);
1333 }
1334
1335 void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio)
1336 {
1337         struct f2fs_sb_info *sbi = fio->sbi;
1338         struct f2fs_summary sum;
1339         struct node_info ni;
1340
1341         f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
1342         get_node_info(sbi, dn->nid, &ni);
1343         set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1344         do_write_page(&sum, fio);
1345         dn->data_blkaddr = fio->blk_addr;
1346 }
1347
1348 void rewrite_data_page(struct f2fs_io_info *fio)
1349 {
1350         stat_inc_inplace_blocks(fio->sbi);
1351         f2fs_submit_page_mbio(fio);
1352 }
1353
1354 static void __f2fs_replace_block(struct f2fs_sb_info *sbi,
1355                                 struct f2fs_summary *sum,
1356                                 block_t old_blkaddr, block_t new_blkaddr,
1357                                 bool recover_curseg)
1358 {
1359         struct sit_info *sit_i = SIT_I(sbi);
1360         struct curseg_info *curseg;
1361         unsigned int segno, old_cursegno;
1362         struct seg_entry *se;
1363         int type;
1364         unsigned short old_blkoff;
1365
1366         segno = GET_SEGNO(sbi, new_blkaddr);
1367         se = get_seg_entry(sbi, segno);
1368         type = se->type;
1369
1370         if (!recover_curseg) {
1371                 /* for recovery flow */
1372                 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
1373                         if (old_blkaddr == NULL_ADDR)
1374                                 type = CURSEG_COLD_DATA;
1375                         else
1376                                 type = CURSEG_WARM_DATA;
1377                 }
1378         } else {
1379                 if (!IS_CURSEG(sbi, segno))
1380                         type = CURSEG_WARM_DATA;
1381         }
1382
1383         curseg = CURSEG_I(sbi, type);
1384
1385         mutex_lock(&curseg->curseg_mutex);
1386         mutex_lock(&sit_i->sentry_lock);
1387
1388         old_cursegno = curseg->segno;
1389         old_blkoff = curseg->next_blkoff;
1390
1391         /* change the current segment */
1392         if (segno != curseg->segno) {
1393                 curseg->next_segno = segno;
1394                 change_curseg(sbi, type, true);
1395         }
1396
1397         curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
1398         __add_sum_entry(sbi, type, sum);
1399
1400         if (!recover_curseg)
1401                 update_sit_entry(sbi, new_blkaddr, 1);
1402         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1403                 update_sit_entry(sbi, old_blkaddr, -1);
1404
1405         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
1406         locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
1407
1408         locate_dirty_segment(sbi, old_cursegno);
1409
1410         if (recover_curseg) {
1411                 if (old_cursegno != curseg->segno) {
1412                         curseg->next_segno = old_cursegno;
1413                         change_curseg(sbi, type, true);
1414                 }
1415                 curseg->next_blkoff = old_blkoff;
1416         }
1417
1418         mutex_unlock(&sit_i->sentry_lock);
1419         mutex_unlock(&curseg->curseg_mutex);
1420 }
1421
1422 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
1423                                 block_t old_addr, block_t new_addr,
1424                                 unsigned char version, bool recover_curseg)
1425 {
1426         struct f2fs_summary sum;
1427
1428         set_summary(&sum, dn->nid, dn->ofs_in_node, version);
1429
1430         __f2fs_replace_block(sbi, &sum, old_addr, new_addr, recover_curseg);
1431
1432         dn->data_blkaddr = new_addr;
1433         set_data_blkaddr(dn);
1434         f2fs_update_extent_cache(dn);
1435 }
1436
1437 static inline bool is_merged_page(struct f2fs_sb_info *sbi,
1438                                         struct page *page, enum page_type type)
1439 {
1440         enum page_type btype = PAGE_TYPE_OF_BIO(type);
1441         struct f2fs_bio_info *io = &sbi->write_io[btype];
1442         struct bio_vec *bvec;
1443         struct page *target;
1444         int i;
1445
1446         down_read(&io->io_rwsem);
1447         if (!io->bio) {
1448                 up_read(&io->io_rwsem);
1449                 return false;
1450         }
1451
1452         bio_for_each_segment_all(bvec, io->bio, i) {
1453
1454                 if (bvec->bv_page->mapping) {
1455                         target = bvec->bv_page;
1456                 } else {
1457                         struct f2fs_crypto_ctx *ctx;
1458
1459                         /* encrypted page */
1460                         ctx = (struct f2fs_crypto_ctx *)page_private(
1461                                                                 bvec->bv_page);
1462                         target = ctx->w.control_page;
1463                 }
1464
1465                 if (page == target) {
1466                         up_read(&io->io_rwsem);
1467                         return true;
1468                 }
1469         }
1470
1471         up_read(&io->io_rwsem);
1472         return false;
1473 }
1474
1475 void f2fs_wait_on_page_writeback(struct page *page,
1476                                 enum page_type type)
1477 {
1478         if (PageWriteback(page)) {
1479                 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
1480
1481                 if (is_merged_page(sbi, page, type))
1482                         f2fs_submit_merged_bio(sbi, type, WRITE);
1483                 wait_on_page_writeback(page);
1484         }
1485 }
1486
1487 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1488 {
1489         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1490         struct curseg_info *seg_i;
1491         unsigned char *kaddr;
1492         struct page *page;
1493         block_t start;
1494         int i, j, offset;
1495
1496         start = start_sum_block(sbi);
1497
1498         page = get_meta_page(sbi, start++);
1499         kaddr = (unsigned char *)page_address(page);
1500
1501         /* Step 1: restore nat cache */
1502         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1503         memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE);
1504
1505         /* Step 2: restore sit cache */
1506         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1507         memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE,
1508                                                 SUM_JOURNAL_SIZE);
1509         offset = 2 * SUM_JOURNAL_SIZE;
1510
1511         /* Step 3: restore summary entries */
1512         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1513                 unsigned short blk_off;
1514                 unsigned int segno;
1515
1516                 seg_i = CURSEG_I(sbi, i);
1517                 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1518                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1519                 seg_i->next_segno = segno;
1520                 reset_curseg(sbi, i, 0);
1521                 seg_i->alloc_type = ckpt->alloc_type[i];
1522                 seg_i->next_blkoff = blk_off;
1523
1524                 if (seg_i->alloc_type == SSR)
1525                         blk_off = sbi->blocks_per_seg;
1526
1527                 for (j = 0; j < blk_off; j++) {
1528                         struct f2fs_summary *s;
1529                         s = (struct f2fs_summary *)(kaddr + offset);
1530                         seg_i->sum_blk->entries[j] = *s;
1531                         offset += SUMMARY_SIZE;
1532                         if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1533                                                 SUM_FOOTER_SIZE)
1534                                 continue;
1535
1536                         f2fs_put_page(page, 1);
1537                         page = NULL;
1538
1539                         page = get_meta_page(sbi, start++);
1540                         kaddr = (unsigned char *)page_address(page);
1541                         offset = 0;
1542                 }
1543         }
1544         f2fs_put_page(page, 1);
1545         return 0;
1546 }
1547
1548 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1549 {
1550         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1551         struct f2fs_summary_block *sum;
1552         struct curseg_info *curseg;
1553         struct page *new;
1554         unsigned short blk_off;
1555         unsigned int segno = 0;
1556         block_t blk_addr = 0;
1557
1558         /* get segment number and block addr */
1559         if (IS_DATASEG(type)) {
1560                 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1561                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1562                                                         CURSEG_HOT_DATA]);
1563                 if (__exist_node_summaries(sbi))
1564                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1565                 else
1566                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1567         } else {
1568                 segno = le32_to_cpu(ckpt->cur_node_segno[type -
1569                                                         CURSEG_HOT_NODE]);
1570                 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1571                                                         CURSEG_HOT_NODE]);
1572                 if (__exist_node_summaries(sbi))
1573                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1574                                                         type - CURSEG_HOT_NODE);
1575                 else
1576                         blk_addr = GET_SUM_BLOCK(sbi, segno);
1577         }
1578
1579         new = get_meta_page(sbi, blk_addr);
1580         sum = (struct f2fs_summary_block *)page_address(new);
1581
1582         if (IS_NODESEG(type)) {
1583                 if (__exist_node_summaries(sbi)) {
1584                         struct f2fs_summary *ns = &sum->entries[0];
1585                         int i;
1586                         for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1587                                 ns->version = 0;
1588                                 ns->ofs_in_node = 0;
1589                         }
1590                 } else {
1591                         int err;
1592
1593                         err = restore_node_summary(sbi, segno, sum);
1594                         if (err) {
1595                                 f2fs_put_page(new, 1);
1596                                 return err;
1597                         }
1598                 }
1599         }
1600
1601         /* set uncompleted segment to curseg */
1602         curseg = CURSEG_I(sbi, type);
1603         mutex_lock(&curseg->curseg_mutex);
1604         memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE);
1605         curseg->next_segno = segno;
1606         reset_curseg(sbi, type, 0);
1607         curseg->alloc_type = ckpt->alloc_type[type];
1608         curseg->next_blkoff = blk_off;
1609         mutex_unlock(&curseg->curseg_mutex);
1610         f2fs_put_page(new, 1);
1611         return 0;
1612 }
1613
1614 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1615 {
1616         int type = CURSEG_HOT_DATA;
1617         int err;
1618
1619         if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {
1620                 int npages = npages_for_summary_flush(sbi, true);
1621
1622                 if (npages >= 2)
1623                         ra_meta_pages(sbi, start_sum_block(sbi), npages,
1624                                                         META_CP, true);
1625
1626                 /* restore for compacted data summary */
1627                 if (read_compacted_summaries(sbi))
1628                         return -EINVAL;
1629                 type = CURSEG_HOT_NODE;
1630         }
1631
1632         if (__exist_node_summaries(sbi))
1633                 ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
1634                                         NR_CURSEG_TYPE - type, META_CP, true);
1635
1636         for (; type <= CURSEG_COLD_NODE; type++) {
1637                 err = read_normal_summaries(sbi, type);
1638                 if (err)
1639                         return err;
1640         }
1641
1642         return 0;
1643 }
1644
1645 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1646 {
1647         struct page *page;
1648         unsigned char *kaddr;
1649         struct f2fs_summary *summary;
1650         struct curseg_info *seg_i;
1651         int written_size = 0;
1652         int i, j;
1653
1654         page = grab_meta_page(sbi, blkaddr++);
1655         kaddr = (unsigned char *)page_address(page);
1656
1657         /* Step 1: write nat cache */
1658         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1659         memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE);
1660         written_size += SUM_JOURNAL_SIZE;
1661
1662         /* Step 2: write sit cache */
1663         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1664         memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits,
1665                                                 SUM_JOURNAL_SIZE);
1666         written_size += SUM_JOURNAL_SIZE;
1667
1668         /* Step 3: write summary entries */
1669         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1670                 unsigned short blkoff;
1671                 seg_i = CURSEG_I(sbi, i);
1672                 if (sbi->ckpt->alloc_type[i] == SSR)
1673                         blkoff = sbi->blocks_per_seg;
1674                 else
1675                         blkoff = curseg_blkoff(sbi, i);
1676
1677                 for (j = 0; j < blkoff; j++) {
1678                         if (!page) {
1679                                 page = grab_meta_page(sbi, blkaddr++);
1680                                 kaddr = (unsigned char *)page_address(page);
1681                                 written_size = 0;
1682                         }
1683                         summary = (struct f2fs_summary *)(kaddr + written_size);
1684                         *summary = seg_i->sum_blk->entries[j];
1685                         written_size += SUMMARY_SIZE;
1686
1687                         if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1688                                                         SUM_FOOTER_SIZE)
1689                                 continue;
1690
1691                         set_page_dirty(page);
1692                         f2fs_put_page(page, 1);
1693                         page = NULL;
1694                 }
1695         }
1696         if (page) {
1697                 set_page_dirty(page);
1698                 f2fs_put_page(page, 1);
1699         }
1700 }
1701
1702 static void write_normal_summaries(struct f2fs_sb_info *sbi,
1703                                         block_t blkaddr, int type)
1704 {
1705         int i, end;
1706         if (IS_DATASEG(type))
1707                 end = type + NR_CURSEG_DATA_TYPE;
1708         else
1709                 end = type + NR_CURSEG_NODE_TYPE;
1710
1711         for (i = type; i < end; i++) {
1712                 struct curseg_info *sum = CURSEG_I(sbi, i);
1713                 mutex_lock(&sum->curseg_mutex);
1714                 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type));
1715                 mutex_unlock(&sum->curseg_mutex);
1716         }
1717 }
1718
1719 void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1720 {
1721         if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))
1722                 write_compacted_summaries(sbi, start_blk);
1723         else
1724                 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
1725 }
1726
1727 void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1728 {
1729         write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
1730 }
1731
1732 int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type,
1733                                         unsigned int val, int alloc)
1734 {
1735         int i;
1736
1737         if (type == NAT_JOURNAL) {
1738                 for (i = 0; i < nats_in_cursum(sum); i++) {
1739                         if (le32_to_cpu(nid_in_journal(sum, i)) == val)
1740                                 return i;
1741                 }
1742                 if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES)
1743                         return update_nats_in_cursum(sum, 1);
1744         } else if (type == SIT_JOURNAL) {
1745                 for (i = 0; i < sits_in_cursum(sum); i++)
1746                         if (le32_to_cpu(segno_in_journal(sum, i)) == val)
1747                                 return i;
1748                 if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES)
1749                         return update_sits_in_cursum(sum, 1);
1750         }
1751         return -1;
1752 }
1753
1754 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
1755                                         unsigned int segno)
1756 {
1757         return get_meta_page(sbi, current_sit_addr(sbi, segno));
1758 }
1759
1760 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
1761                                         unsigned int start)
1762 {
1763         struct sit_info *sit_i = SIT_I(sbi);
1764         struct page *src_page, *dst_page;
1765         pgoff_t src_off, dst_off;
1766         void *src_addr, *dst_addr;
1767
1768         src_off = current_sit_addr(sbi, start);
1769         dst_off = next_sit_addr(sbi, src_off);
1770
1771         /* get current sit block page without lock */
1772         src_page = get_meta_page(sbi, src_off);
1773         dst_page = grab_meta_page(sbi, dst_off);
1774         f2fs_bug_on(sbi, PageDirty(src_page));
1775
1776         src_addr = page_address(src_page);
1777         dst_addr = page_address(dst_page);
1778         memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
1779
1780         set_page_dirty(dst_page);
1781         f2fs_put_page(src_page, 1);
1782
1783         set_to_next_sit(sit_i, start);
1784
1785         return dst_page;
1786 }
1787
1788 static struct sit_entry_set *grab_sit_entry_set(void)
1789 {
1790         struct sit_entry_set *ses =
1791                         f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
1792
1793         ses->entry_cnt = 0;
1794         INIT_LIST_HEAD(&ses->set_list);
1795         return ses;
1796 }
1797
1798 static void release_sit_entry_set(struct sit_entry_set *ses)
1799 {
1800         list_del(&ses->set_list);
1801         kmem_cache_free(sit_entry_set_slab, ses);
1802 }
1803
1804 static void adjust_sit_entry_set(struct sit_entry_set *ses,
1805                                                 struct list_head *head)
1806 {
1807         struct sit_entry_set *next = ses;
1808
1809         if (list_is_last(&ses->set_list, head))
1810                 return;
1811
1812         list_for_each_entry_continue(next, head, set_list)
1813                 if (ses->entry_cnt <= next->entry_cnt)
1814                         break;
1815
1816         list_move_tail(&ses->set_list, &next->set_list);
1817 }
1818
1819 static void add_sit_entry(unsigned int segno, struct list_head *head)
1820 {
1821         struct sit_entry_set *ses;
1822         unsigned int start_segno = START_SEGNO(segno);
1823
1824         list_for_each_entry(ses, head, set_list) {
1825                 if (ses->start_segno == start_segno) {
1826                         ses->entry_cnt++;
1827                         adjust_sit_entry_set(ses, head);
1828                         return;
1829                 }
1830         }
1831
1832         ses = grab_sit_entry_set();
1833
1834         ses->start_segno = start_segno;
1835         ses->entry_cnt++;
1836         list_add(&ses->set_list, head);
1837 }
1838
1839 static void add_sits_in_set(struct f2fs_sb_info *sbi)
1840 {
1841         struct f2fs_sm_info *sm_info = SM_I(sbi);
1842         struct list_head *set_list = &sm_info->sit_entry_set;
1843         unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
1844         unsigned int segno;
1845
1846         for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
1847                 add_sit_entry(segno, set_list);
1848 }
1849
1850 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
1851 {
1852         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1853         struct f2fs_summary_block *sum = curseg->sum_blk;
1854         int i;
1855
1856         for (i = sits_in_cursum(sum) - 1; i >= 0; i--) {
1857                 unsigned int segno;
1858                 bool dirtied;
1859
1860                 segno = le32_to_cpu(segno_in_journal(sum, i));
1861                 dirtied = __mark_sit_entry_dirty(sbi, segno);
1862
1863                 if (!dirtied)
1864                         add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
1865         }
1866         update_sits_in_cursum(sum, -sits_in_cursum(sum));
1867 }
1868
1869 /*
1870  * CP calls this function, which flushes SIT entries including sit_journal,
1871  * and moves prefree segs to free segs.
1872  */
1873 void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1874 {
1875         struct sit_info *sit_i = SIT_I(sbi);
1876         unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
1877         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1878         struct f2fs_summary_block *sum = curseg->sum_blk;
1879         struct sit_entry_set *ses, *tmp;
1880         struct list_head *head = &SM_I(sbi)->sit_entry_set;
1881         bool to_journal = true;
1882         struct seg_entry *se;
1883
1884         mutex_lock(&curseg->curseg_mutex);
1885         mutex_lock(&sit_i->sentry_lock);
1886
1887         if (!sit_i->dirty_sentries)
1888                 goto out;
1889
1890         /*
1891          * add and account sit entries of dirty bitmap in sit entry
1892          * set temporarily
1893          */
1894         add_sits_in_set(sbi);
1895
1896         /*
1897          * if there are no enough space in journal to store dirty sit
1898          * entries, remove all entries from journal and add and account
1899          * them in sit entry set.
1900          */
1901         if (!__has_cursum_space(sum, sit_i->dirty_sentries, SIT_JOURNAL))
1902                 remove_sits_in_journal(sbi);
1903
1904         /*
1905          * there are two steps to flush sit entries:
1906          * #1, flush sit entries to journal in current cold data summary block.
1907          * #2, flush sit entries to sit page.
1908          */
1909         list_for_each_entry_safe(ses, tmp, head, set_list) {
1910                 struct page *page = NULL;
1911                 struct f2fs_sit_block *raw_sit = NULL;
1912                 unsigned int start_segno = ses->start_segno;
1913                 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
1914                                                 (unsigned long)MAIN_SEGS(sbi));
1915                 unsigned int segno = start_segno;
1916
1917                 if (to_journal &&
1918                         !__has_cursum_space(sum, ses->entry_cnt, SIT_JOURNAL))
1919                         to_journal = false;
1920
1921                 if (!to_journal) {
1922                         page = get_next_sit_page(sbi, start_segno);
1923                         raw_sit = page_address(page);
1924                 }
1925
1926                 /* flush dirty sit entries in region of current sit set */
1927                 for_each_set_bit_from(segno, bitmap, end) {
1928                         int offset, sit_offset;
1929
1930                         se = get_seg_entry(sbi, segno);
1931
1932                         /* add discard candidates */
1933                         if (cpc->reason != CP_DISCARD) {
1934                                 cpc->trim_start = segno;
1935                                 add_discard_addrs(sbi, cpc);
1936                         }
1937
1938                         if (to_journal) {
1939                                 offset = lookup_journal_in_cursum(sum,
1940                                                         SIT_JOURNAL, segno, 1);
1941                                 f2fs_bug_on(sbi, offset < 0);
1942                                 segno_in_journal(sum, offset) =
1943                                                         cpu_to_le32(segno);
1944                                 seg_info_to_raw_sit(se,
1945                                                 &sit_in_journal(sum, offset));
1946                         } else {
1947                                 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
1948                                 seg_info_to_raw_sit(se,
1949                                                 &raw_sit->entries[sit_offset]);
1950                         }
1951
1952                         __clear_bit(segno, bitmap);
1953                         sit_i->dirty_sentries--;
1954                         ses->entry_cnt--;
1955                 }
1956
1957                 if (!to_journal)
1958                         f2fs_put_page(page, 1);
1959
1960                 f2fs_bug_on(sbi, ses->entry_cnt);
1961                 release_sit_entry_set(ses);
1962         }
1963
1964         f2fs_bug_on(sbi, !list_empty(head));
1965         f2fs_bug_on(sbi, sit_i->dirty_sentries);
1966 out:
1967         if (cpc->reason == CP_DISCARD) {
1968                 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
1969                         add_discard_addrs(sbi, cpc);
1970         }
1971         mutex_unlock(&sit_i->sentry_lock);
1972         mutex_unlock(&curseg->curseg_mutex);
1973
1974         set_prefree_as_free_segments(sbi);
1975 }
1976
1977 static int build_sit_info(struct f2fs_sb_info *sbi)
1978 {
1979         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
1980         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1981         struct sit_info *sit_i;
1982         unsigned int sit_segs, start;
1983         char *src_bitmap, *dst_bitmap;
1984         unsigned int bitmap_size;
1985
1986         /* allocate memory for SIT information */
1987         sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
1988         if (!sit_i)
1989                 return -ENOMEM;
1990
1991         SM_I(sbi)->sit_info = sit_i;
1992
1993         sit_i->sentries = f2fs_kvzalloc(MAIN_SEGS(sbi) *
1994                                         sizeof(struct seg_entry), GFP_KERNEL);
1995         if (!sit_i->sentries)
1996                 return -ENOMEM;
1997
1998         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
1999         sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2000         if (!sit_i->dirty_sentries_bitmap)
2001                 return -ENOMEM;
2002
2003         for (start = 0; start < MAIN_SEGS(sbi); start++) {
2004                 sit_i->sentries[start].cur_valid_map
2005                         = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2006                 sit_i->sentries[start].ckpt_valid_map
2007                         = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2008                 sit_i->sentries[start].discard_map
2009                         = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2010                 if (!sit_i->sentries[start].cur_valid_map ||
2011                                 !sit_i->sentries[start].ckpt_valid_map ||
2012                                 !sit_i->sentries[start].discard_map)
2013                         return -ENOMEM;
2014         }
2015
2016         sit_i->tmp_map = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2017         if (!sit_i->tmp_map)
2018                 return -ENOMEM;
2019
2020         if (sbi->segs_per_sec > 1) {
2021                 sit_i->sec_entries = f2fs_kvzalloc(MAIN_SECS(sbi) *
2022                                         sizeof(struct sec_entry), GFP_KERNEL);
2023                 if (!sit_i->sec_entries)
2024                         return -ENOMEM;
2025         }
2026
2027         /* get information related with SIT */
2028         sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
2029
2030         /* setup SIT bitmap from ckeckpoint pack */
2031         bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
2032         src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
2033
2034         dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
2035         if (!dst_bitmap)
2036                 return -ENOMEM;
2037
2038         /* init SIT information */
2039         sit_i->s_ops = &default_salloc_ops;
2040
2041         sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
2042         sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
2043         sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
2044         sit_i->sit_bitmap = dst_bitmap;
2045         sit_i->bitmap_size = bitmap_size;
2046         sit_i->dirty_sentries = 0;
2047         sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
2048         sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
2049         sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
2050         mutex_init(&sit_i->sentry_lock);
2051         return 0;
2052 }
2053
2054 static int build_free_segmap(struct f2fs_sb_info *sbi)
2055 {
2056         struct free_segmap_info *free_i;
2057         unsigned int bitmap_size, sec_bitmap_size;
2058
2059         /* allocate memory for free segmap information */
2060         free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
2061         if (!free_i)
2062                 return -ENOMEM;
2063
2064         SM_I(sbi)->free_info = free_i;
2065
2066         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2067         free_i->free_segmap = f2fs_kvmalloc(bitmap_size, GFP_KERNEL);
2068         if (!free_i->free_segmap)
2069                 return -ENOMEM;
2070
2071         sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2072         free_i->free_secmap = f2fs_kvmalloc(sec_bitmap_size, GFP_KERNEL);
2073         if (!free_i->free_secmap)
2074                 return -ENOMEM;
2075
2076         /* set all segments as dirty temporarily */
2077         memset(free_i->free_segmap, 0xff, bitmap_size);
2078         memset(free_i->free_secmap, 0xff, sec_bitmap_size);
2079
2080         /* init free segmap information */
2081         free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
2082         free_i->free_segments = 0;
2083         free_i->free_sections = 0;
2084         spin_lock_init(&free_i->segmap_lock);
2085         return 0;
2086 }
2087
2088 static int build_curseg(struct f2fs_sb_info *sbi)
2089 {
2090         struct curseg_info *array;
2091         int i;
2092
2093         array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL);
2094         if (!array)
2095                 return -ENOMEM;
2096
2097         SM_I(sbi)->curseg_array = array;
2098
2099         for (i = 0; i < NR_CURSEG_TYPE; i++) {
2100                 mutex_init(&array[i].curseg_mutex);
2101                 array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL);
2102                 if (!array[i].sum_blk)
2103                         return -ENOMEM;
2104                 array[i].segno = NULL_SEGNO;
2105                 array[i].next_blkoff = 0;
2106         }
2107         return restore_curseg_summaries(sbi);
2108 }
2109
2110 static void build_sit_entries(struct f2fs_sb_info *sbi)
2111 {
2112         struct sit_info *sit_i = SIT_I(sbi);
2113         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
2114         struct f2fs_summary_block *sum = curseg->sum_blk;
2115         int sit_blk_cnt = SIT_BLK_CNT(sbi);
2116         unsigned int i, start, end;
2117         unsigned int readed, start_blk = 0;
2118         int nrpages = MAX_BIO_BLOCKS(sbi);
2119
2120         do {
2121                 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true);
2122
2123                 start = start_blk * sit_i->sents_per_block;
2124                 end = (start_blk + readed) * sit_i->sents_per_block;
2125
2126                 for (; start < end && start < MAIN_SEGS(sbi); start++) {
2127                         struct seg_entry *se = &sit_i->sentries[start];
2128                         struct f2fs_sit_block *sit_blk;
2129                         struct f2fs_sit_entry sit;
2130                         struct page *page;
2131
2132                         mutex_lock(&curseg->curseg_mutex);
2133                         for (i = 0; i < sits_in_cursum(sum); i++) {
2134                                 if (le32_to_cpu(segno_in_journal(sum, i))
2135                                                                 == start) {
2136                                         sit = sit_in_journal(sum, i);
2137                                         mutex_unlock(&curseg->curseg_mutex);
2138                                         goto got_it;
2139                                 }
2140                         }
2141                         mutex_unlock(&curseg->curseg_mutex);
2142
2143                         page = get_current_sit_page(sbi, start);
2144                         sit_blk = (struct f2fs_sit_block *)page_address(page);
2145                         sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
2146                         f2fs_put_page(page, 1);
2147 got_it:
2148                         check_block_count(sbi, start, &sit);
2149                         seg_info_from_raw_sit(se, &sit);
2150
2151                         /* build discard map only one time */
2152                         memcpy(se->discard_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE);
2153                         sbi->discard_blks += sbi->blocks_per_seg - se->valid_blocks;
2154
2155                         if (sbi->segs_per_sec > 1) {
2156                                 struct sec_entry *e = get_sec_entry(sbi, start);
2157                                 e->valid_blocks += se->valid_blocks;
2158                         }
2159                 }
2160                 start_blk += readed;
2161         } while (start_blk < sit_blk_cnt);
2162 }
2163
2164 static void init_free_segmap(struct f2fs_sb_info *sbi)
2165 {
2166         unsigned int start;
2167         int type;
2168
2169         for (start = 0; start < MAIN_SEGS(sbi); start++) {
2170                 struct seg_entry *sentry = get_seg_entry(sbi, start);
2171                 if (!sentry->valid_blocks)
2172                         __set_free(sbi, start);
2173         }
2174
2175         /* set use the current segments */
2176         for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
2177                 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
2178                 __set_test_and_inuse(sbi, curseg_t->segno);
2179         }
2180 }
2181
2182 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
2183 {
2184         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2185         struct free_segmap_info *free_i = FREE_I(sbi);
2186         unsigned int segno = 0, offset = 0;
2187         unsigned short valid_blocks;
2188
2189         while (1) {
2190                 /* find dirty segment based on free segmap */
2191                 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
2192                 if (segno >= MAIN_SEGS(sbi))
2193                         break;
2194                 offset = segno + 1;
2195                 valid_blocks = get_valid_blocks(sbi, segno, 0);
2196                 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks)
2197                         continue;
2198                 if (valid_blocks > sbi->blocks_per_seg) {
2199                         f2fs_bug_on(sbi, 1);
2200                         continue;
2201                 }
2202                 mutex_lock(&dirty_i->seglist_lock);
2203                 __locate_dirty_segment(sbi, segno, DIRTY);
2204                 mutex_unlock(&dirty_i->seglist_lock);
2205         }
2206 }
2207
2208 static int init_victim_secmap(struct f2fs_sb_info *sbi)
2209 {
2210         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2211         unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2212
2213         dirty_i->victim_secmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2214         if (!dirty_i->victim_secmap)
2215                 return -ENOMEM;
2216         return 0;
2217 }
2218
2219 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
2220 {
2221         struct dirty_seglist_info *dirty_i;
2222         unsigned int bitmap_size, i;
2223
2224         /* allocate memory for dirty segments list information */
2225         dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
2226         if (!dirty_i)
2227                 return -ENOMEM;
2228
2229         SM_I(sbi)->dirty_info = dirty_i;
2230         mutex_init(&dirty_i->seglist_lock);
2231
2232         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2233
2234         for (i = 0; i < NR_DIRTY_TYPE; i++) {
2235                 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2236                 if (!dirty_i->dirty_segmap[i])
2237                         return -ENOMEM;
2238         }
2239
2240         init_dirty_segmap(sbi);
2241         return init_victim_secmap(sbi);
2242 }
2243
2244 /*
2245  * Update min, max modified time for cost-benefit GC algorithm
2246  */
2247 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
2248 {
2249         struct sit_info *sit_i = SIT_I(sbi);
2250         unsigned int segno;
2251
2252         mutex_lock(&sit_i->sentry_lock);
2253
2254         sit_i->min_mtime = LLONG_MAX;
2255
2256         for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
2257                 unsigned int i;
2258                 unsigned long long mtime = 0;
2259
2260                 for (i = 0; i < sbi->segs_per_sec; i++)
2261                         mtime += get_seg_entry(sbi, segno + i)->mtime;
2262
2263                 mtime = div_u64(mtime, sbi->segs_per_sec);
2264
2265                 if (sit_i->min_mtime > mtime)
2266                         sit_i->min_mtime = mtime;
2267         }
2268         sit_i->max_mtime = get_mtime(sbi);
2269         mutex_unlock(&sit_i->sentry_lock);
2270 }
2271
2272 int build_segment_manager(struct f2fs_sb_info *sbi)
2273 {
2274         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2275         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2276         struct f2fs_sm_info *sm_info;
2277         int err;
2278
2279         sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
2280         if (!sm_info)
2281                 return -ENOMEM;
2282
2283         /* init sm info */
2284         sbi->sm_info = sm_info;
2285         sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
2286         sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
2287         sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
2288         sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
2289         sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
2290         sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
2291         sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
2292         sm_info->rec_prefree_segments = sm_info->main_segments *
2293                                         DEF_RECLAIM_PREFREE_SEGMENTS / 100;
2294         sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
2295         sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
2296         sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
2297
2298         INIT_LIST_HEAD(&sm_info->discard_list);
2299         sm_info->nr_discards = 0;
2300         sm_info->max_discards = 0;
2301
2302         sm_info->trim_sections = DEF_BATCHED_TRIM_SECTIONS;
2303
2304         INIT_LIST_HEAD(&sm_info->sit_entry_set);
2305
2306         if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) {
2307                 err = create_flush_cmd_control(sbi);
2308                 if (err)
2309                         return err;
2310         }
2311
2312         err = build_sit_info(sbi);
2313         if (err)
2314                 return err;
2315         err = build_free_segmap(sbi);
2316         if (err)
2317                 return err;
2318         err = build_curseg(sbi);
2319         if (err)
2320                 return err;
2321
2322         /* reinit free segmap based on SIT */
2323         build_sit_entries(sbi);
2324
2325         init_free_segmap(sbi);
2326         err = build_dirty_segmap(sbi);
2327         if (err)
2328                 return err;
2329
2330         init_min_max_mtime(sbi);
2331         return 0;
2332 }
2333
2334 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
2335                 enum dirty_type dirty_type)
2336 {
2337         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2338
2339         mutex_lock(&dirty_i->seglist_lock);
2340         kvfree(dirty_i->dirty_segmap[dirty_type]);
2341         dirty_i->nr_dirty[dirty_type] = 0;
2342         mutex_unlock(&dirty_i->seglist_lock);
2343 }
2344
2345 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
2346 {
2347         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2348         kvfree(dirty_i->victim_secmap);
2349 }
2350
2351 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
2352 {
2353         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2354         int i;
2355
2356         if (!dirty_i)
2357                 return;
2358
2359         /* discard pre-free/dirty segments list */
2360         for (i = 0; i < NR_DIRTY_TYPE; i++)
2361                 discard_dirty_segmap(sbi, i);
2362
2363         destroy_victim_secmap(sbi);
2364         SM_I(sbi)->dirty_info = NULL;
2365         kfree(dirty_i);
2366 }
2367
2368 static void destroy_curseg(struct f2fs_sb_info *sbi)
2369 {
2370         struct curseg_info *array = SM_I(sbi)->curseg_array;
2371         int i;
2372
2373         if (!array)
2374                 return;
2375         SM_I(sbi)->curseg_array = NULL;
2376         for (i = 0; i < NR_CURSEG_TYPE; i++)
2377                 kfree(array[i].sum_blk);
2378         kfree(array);
2379 }
2380
2381 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
2382 {
2383         struct free_segmap_info *free_i = SM_I(sbi)->free_info;
2384         if (!free_i)
2385                 return;
2386         SM_I(sbi)->free_info = NULL;
2387         kvfree(free_i->free_segmap);
2388         kvfree(free_i->free_secmap);
2389         kfree(free_i);
2390 }
2391
2392 static void destroy_sit_info(struct f2fs_sb_info *sbi)
2393 {
2394         struct sit_info *sit_i = SIT_I(sbi);
2395         unsigned int start;
2396
2397         if (!sit_i)
2398                 return;
2399
2400         if (sit_i->sentries) {
2401                 for (start = 0; start < MAIN_SEGS(sbi); start++) {
2402                         kfree(sit_i->sentries[start].cur_valid_map);
2403                         kfree(sit_i->sentries[start].ckpt_valid_map);
2404                         kfree(sit_i->sentries[start].discard_map);
2405                 }
2406         }
2407         kfree(sit_i->tmp_map);
2408
2409         kvfree(sit_i->sentries);
2410         kvfree(sit_i->sec_entries);
2411         kvfree(sit_i->dirty_sentries_bitmap);
2412
2413         SM_I(sbi)->sit_info = NULL;
2414         kfree(sit_i->sit_bitmap);
2415         kfree(sit_i);
2416 }
2417
2418 void destroy_segment_manager(struct f2fs_sb_info *sbi)
2419 {
2420         struct f2fs_sm_info *sm_info = SM_I(sbi);
2421
2422         if (!sm_info)
2423                 return;
2424         destroy_flush_cmd_control(sbi);
2425         destroy_dirty_segmap(sbi);
2426         destroy_curseg(sbi);
2427         destroy_free_segmap(sbi);
2428         destroy_sit_info(sbi);
2429         sbi->sm_info = NULL;
2430         kfree(sm_info);
2431 }
2432
2433 int __init create_segment_manager_caches(void)
2434 {
2435         discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
2436                         sizeof(struct discard_entry));
2437         if (!discard_entry_slab)
2438                 goto fail;
2439
2440         sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
2441                         sizeof(struct sit_entry_set));
2442         if (!sit_entry_set_slab)
2443                 goto destory_discard_entry;
2444
2445         inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry",
2446                         sizeof(struct inmem_pages));
2447         if (!inmem_entry_slab)
2448                 goto destroy_sit_entry_set;
2449         return 0;
2450
2451 destroy_sit_entry_set:
2452         kmem_cache_destroy(sit_entry_set_slab);
2453 destory_discard_entry:
2454         kmem_cache_destroy(discard_entry_slab);
2455 fail:
2456         return -ENOMEM;
2457 }
2458
2459 void destroy_segment_manager_caches(void)
2460 {
2461         kmem_cache_destroy(sit_entry_set_slab);
2462         kmem_cache_destroy(discard_entry_slab);
2463         kmem_cache_destroy(inmem_entry_slab);
2464 }