kernel/power/snapshot.c

   1 /*
   2  * linux/kernel/power/snapshot.c
   3  *
   4  * This file provide system snapshot/restore functionality.
   5  *
   6  * Copyright (C) 1998-2005 Pavel Machek <pavel@suse.cz>
   7  *
   8  * This file is released under the GPLv2, and is based on swsusp.c.
   9  *
  10  */
  11
  12
  13 #include <linux/module.h>
  14 #include <linux/mm.h>
  15 #include <linux/suspend.h>
  16 #include <linux/smp_lock.h>
  17 #include <linux/delay.h>
  18 #include <linux/bitops.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/kernel.h>
  21 #include <linux/pm.h>
  22 #include <linux/device.h>
  23 #include <linux/bootmem.h>
  24 #include <linux/syscalls.h>
  25 #include <linux/console.h>
  26 #include <linux/highmem.h>
  27
  28 #include <asm/uaccess.h>
  29 #include <asm/mmu_context.h>
  30 #include <asm/pgtable.h>
  31 #include <asm/tlbflush.h>
  32 #include <asm/io.h>
  33
  34 #include "power.h"
  35
  36 struct pbe *pagedir_nosave;
  37 unsigned int nr_copy_pages;
  38
  39 #ifdef CONFIG_HIGHMEM
  40 struct highmem_page {
  41         char *data;
  42         struct page *page;
  43         struct highmem_page *next;
  44 };
  45
  46 static struct highmem_page *highmem_copy;
  47
  48 static int save_highmem_zone(struct zone *zone)
  49 {
  50         unsigned long zone_pfn;
  51         mark_free_pages(zone);
  52         for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) {
  53                 struct page *page;
  54                 struct highmem_page *save;
  55                 void *kaddr;
  56                 unsigned long pfn = zone_pfn + zone->zone_start_pfn;
  57
  58                 if (!(pfn%1000))
  59                         printk(".");
  60                 if (!pfn_valid(pfn))
  61                         continue;
  62                 page = pfn_to_page(pfn);
  63                 /*
  64                  * This condition results from rvmalloc() sans vmalloc_32()
  65                  * and architectural memory reservations. This should be
  66                  * corrected eventually when the cases giving rise to this
  67                  * are better understood.
  68                  */
  69                 if (PageReserved(page)) {
  70                         printk("highmem reserved page?!\n");
  71                         continue;
  72                 }
  73                 BUG_ON(PageNosave(page));
  74                 if (PageNosaveFree(page))
  75                         continue;
  76                 save = kmalloc(sizeof(struct highmem_page), GFP_ATOMIC);
  77                 if (!save)
  78                         return -ENOMEM;
  79                 save->next = highmem_copy;
  80                 save->page = page;
  81                 save->data = (void *) get_zeroed_page(GFP_ATOMIC);
  82                 if (!save->data) {
  83                         kfree(save);
  84                         return -ENOMEM;
  85                 }
  86                 kaddr = kmap_atomic(page, KM_USER0);
  87                 memcpy(save->data, kaddr, PAGE_SIZE);
  88                 kunmap_atomic(kaddr, KM_USER0);
  89                 highmem_copy = save;
  90         }
  91         return 0;
  92 }
  93
  94 int save_highmem(void)
  95 {
  96         struct zone *zone;
  97         int res = 0;
  98
  99         pr_debug("swsusp: Saving Highmem\n");
 100         for_each_zone (zone) {
 101                 if (is_highmem(zone))
 102                         res = save_highmem_zone(zone);
 103                 if (res)
 104                         return res;
 105         }
 106         return 0;
 107 }
 108
 109 int restore_highmem(void)
 110 {
 111         printk("swsusp: Restoring Highmem\n");
 112         while (highmem_copy) {
 113                 struct highmem_page *save = highmem_copy;
 114                 void *kaddr;
 115                 highmem_copy = save->next;
 116
 117                 kaddr = kmap_atomic(save->page, KM_USER0);
 118                 memcpy(kaddr, save->data, PAGE_SIZE);
 119                 kunmap_atomic(kaddr, KM_USER0);
 120                 free_page((long) save->data);
 121                 kfree(save);
 122         }
 123         return 0;
 124 }
 125 #endif
 126
 127 static int pfn_is_nosave(unsigned long pfn)
 128 {
 129         unsigned long nosave_begin_pfn = __pa(&__nosave_begin) >> PAGE_SHIFT;
 130         unsigned long nosave_end_pfn = PAGE_ALIGN(__pa(&__nosave_end)) >> PAGE_SHIFT;
 131         return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn);
 132 }
 133
 134 /**
 135  *      saveable - Determine whether a page should be cloned or not.
 136  *      @pfn:   The page
 137  *
 138  *      We save a page if it's Reserved, and not in the range of pages
 139  *      statically defined as 'unsaveable', or if it isn't reserved, and
 140  *      isn't part of a free chunk of pages.
 141  */
 142
 143 static int saveable(struct zone *zone, unsigned long *zone_pfn)
 144 {
 145         unsigned long pfn = *zone_pfn + zone->zone_start_pfn;
 146         struct page *page;
 147
 148         if (!pfn_valid(pfn))
 149                 return 0;
 150
 151         page = pfn_to_page(pfn);
 152         BUG_ON(PageReserved(page) && PageNosave(page));
 153         if (PageNosave(page))
 154                 return 0;
 155         if (PageReserved(page) && pfn_is_nosave(pfn)) {
 156                 pr_debug("[nosave pfn 0x%lx]", pfn);
 157                 return 0;
 158         }
 159         if (PageNosaveFree(page))
 160                 return 0;
 161
 162         return 1;
 163 }
 164
 165 static unsigned count_data_pages(void)
 166 {
 167         struct zone *zone;
 168         unsigned long zone_pfn;
 169         unsigned int n = 0;
 170
 171         for_each_zone (zone) {
 172                 if (is_highmem(zone))
 173                         continue;
 174                 mark_free_pages(zone);
 175                 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn)
 176                         n += saveable(zone, &zone_pfn);
 177         }
 178         return n;
 179 }
 180
 181 static void copy_data_pages(struct pbe *pblist)
 182 {
 183         struct zone *zone;
 184         unsigned long zone_pfn;
 185         struct pbe *pbe, *p;
 186
 187         pbe = pblist;
 188         for_each_zone (zone) {
 189                 if (is_highmem(zone))
 190                         continue;
 191                 mark_free_pages(zone);
 192                 /* This is necessary for swsusp_free() */
 193                 for_each_pb_page (p, pblist)
 194                         SetPageNosaveFree(virt_to_page(p));
 195                 for_each_pbe (p, pblist)
 196                         SetPageNosaveFree(virt_to_page(p->address));
 197                 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) {
 198                         if (saveable(zone, &zone_pfn)) {
 199                                 struct page *page;
 200                                 page = pfn_to_page(zone_pfn + zone->zone_start_pfn);
 201                                 BUG_ON(!pbe);
 202                                 pbe->orig_address = (unsigned long)page_address(page);
 203                                 /* copy_page is not usable for copying task structs. */
 204                                 memcpy((void *)pbe->address, (void *)pbe->orig_address, PAGE_SIZE);
 205                                 pbe = pbe->next;
 206                         }
 207                 }
 208         }
 209         BUG_ON(pbe);
 210 }
 211
 212
 213 /**
 214  *      free_pagedir - free pages allocated with alloc_pagedir()
 215  */
 216
 217 void free_pagedir(struct pbe *pblist)
 218 {
 219         struct pbe *pbe;
 220
 221         while (pblist) {
 222                 pbe = (pblist + PB_PAGE_SKIP)->next;
 223                 ClearPageNosave(virt_to_page(pblist));
 224                 ClearPageNosaveFree(virt_to_page(pblist));
 225                 free_page((unsigned long)pblist);
 226                 pblist = pbe;
 227         }
 228 }
 229
 230 /**
 231  *      fill_pb_page - Create a list of PBEs on a given memory page
 232  */
 233
 234 static inline void fill_pb_page(struct pbe *pbpage)
 235 {
 236         struct pbe *p;
 237
 238         p = pbpage;
 239         pbpage += PB_PAGE_SKIP;
 240         do
 241                 p->next = p + 1;
 242         while (++p < pbpage);
 243 }
 244
 245 /**
 246  *      create_pbe_list - Create a list of PBEs on top of a given chain
 247  *      of memory pages allocated with alloc_pagedir()
 248  */
 249
 250 static inline void create_pbe_list(struct pbe *pblist, unsigned int nr_pages)
 251 {
 252         struct pbe *pbpage, *p;
 253         unsigned int num = PBES_PER_PAGE;
 254
 255         for_each_pb_page (pbpage, pblist) {
 256                 if (num >= nr_pages)
 257                         break;
 258
 259                 fill_pb_page(pbpage);
 260                 num += PBES_PER_PAGE;
 261         }
 262         if (pbpage) {
 263                 for (num -= PBES_PER_PAGE - 1, p = pbpage; num < nr_pages; p++, num++)
 264                         p->next = p + 1;
 265                 p->next = NULL;
 266         }
 267 }
 268
 269 /**
 270  *      @safe_needed - on resume, for storing the PBE list and the image,
 271  *      we can only use memory pages that do not conflict with the pages
 272  *      which had been used before suspend.
 273  *
 274  *      The unsafe pages are marked with the PG_nosave_free flag
 275  *
 276  *      Allocated but unusable (ie eaten) memory pages should be marked
 277  *      so that swsusp_free() can release them
 278  */
 279
 280 static inline void *alloc_image_page(gfp_t gfp_mask, int safe_needed)
 281 {
 282         void *res;
 283
 284         if (safe_needed)
 285                 do {
 286                         res = (void *)get_zeroed_page(gfp_mask);
 287                         if (res && PageNosaveFree(virt_to_page(res)))
 288                                 /* This is for swsusp_free() */
 289                                 SetPageNosave(virt_to_page(res));
 290                 } while (res && PageNosaveFree(virt_to_page(res)));
 291         else
 292                 res = (void *)get_zeroed_page(gfp_mask);
 293         if (res) {
 294                 SetPageNosave(virt_to_page(res));
 295                 SetPageNosaveFree(virt_to_page(res));
 296         }
 297         return res;
 298 }
 299
 300 unsigned long get_safe_page(gfp_t gfp_mask)
 301 {
 302         return (unsigned long)alloc_image_page(gfp_mask, 1);
 303 }
 304
 305 /**
 306  *      alloc_pagedir - Allocate the page directory.
 307  *
 308  *      First, determine exactly how many pages we need and
 309  *      allocate them.
 310  *
 311  *      We arrange the pages in a chain: each page is an array of PBES_PER_PAGE
 312  *      struct pbe elements (pbes) and the last element in the page points
 313  *      to the next page.
 314  *
 315  *      On each page we set up a list of struct_pbe elements.
 316  */
 317
 318 struct pbe *alloc_pagedir(unsigned int nr_pages, gfp_t gfp_mask, int safe_needed)
 319 {
 320         unsigned int num;
 321         struct pbe *pblist, *pbe;
 322
 323         if (!nr_pages)
 324                 return NULL;
 325
 326         pr_debug("alloc_pagedir(): nr_pages = %d\n", nr_pages);
 327         pblist = alloc_image_page(gfp_mask, safe_needed);
 328         /* FIXME: rewrite this ugly loop */
 329         for (pbe = pblist, num = PBES_PER_PAGE; pbe && num < nr_pages;
 330                         pbe = pbe->next, num += PBES_PER_PAGE) {
 331                 pbe += PB_PAGE_SKIP;
 332                 pbe->next = alloc_image_page(gfp_mask, safe_needed);
 333         }
 334         if (!pbe) { /* get_zeroed_page() failed */
 335                 free_pagedir(pblist);
 336                 pblist = NULL;
 337         } else
 338                 create_pbe_list(pblist, nr_pages);
 339         return pblist;
 340 }
 341
 342 /**
 343  * Free pages we allocated for suspend. Suspend pages are alocated
 344  * before atomic copy, so we need to free them after resume.
 345  */
 346
 347 void swsusp_free(void)
 348 {
 349         struct zone *zone;
 350         unsigned long zone_pfn;
 351
 352         for_each_zone(zone) {
 353                 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn)
 354                         if (pfn_valid(zone_pfn + zone->zone_start_pfn)) {
 355                                 struct page *page;
 356                                 page = pfn_to_page(zone_pfn + zone->zone_start_pfn);
 357                                 if (PageNosave(page) && PageNosaveFree(page)) {
 358                                         ClearPageNosave(page);
 359                                         ClearPageNosaveFree(page);
 360                                         free_page((long) page_address(page));
 361                                 }
 362                         }
 363         }
 364 }
 365
 366
 367 /**
 368  *      enough_free_mem - Make sure we enough free memory to snapshot.
 369  *
 370  *      Returns TRUE or FALSE after checking the number of available
 371  *      free pages.
 372  */
 373
 374 static int enough_free_mem(unsigned int nr_pages)
 375 {
 376         pr_debug("swsusp: available memory: %u pages\n", nr_free_pages());
 377         return nr_free_pages() > (nr_pages + PAGES_FOR_IO +
 378                 (nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE);
 379 }
 380
 381 int alloc_data_pages(struct pbe *pblist, gfp_t gfp_mask, int safe_needed)
 382 {
 383         struct pbe *p;
 384
 385         for_each_pbe (p, pblist) {
 386                 p->address = (unsigned long)alloc_image_page(gfp_mask, safe_needed);
 387                 if (!p->address)
 388                         return -ENOMEM;
 389         }
 390         return 0;
 391 }
 392
 393 static struct pbe *swsusp_alloc(unsigned int nr_pages)
 394 {
 395         struct pbe *pblist;
 396
 397         if (!(pblist = alloc_pagedir(nr_pages, GFP_ATOMIC | __GFP_COLD, 0))) {
 398                 printk(KERN_ERR "suspend: Allocating pagedir failed.\n");
 399                 return NULL;
 400         }
 401
 402         if (alloc_data_pages(pblist, GFP_ATOMIC | __GFP_COLD, 0)) {
 403                 printk(KERN_ERR "suspend: Allocating image pages failed.\n");
 404                 swsusp_free();
 405                 return NULL;
 406         }
 407
 408         return pblist;
 409 }
 410
 411 asmlinkage int swsusp_save(void)
 412 {
 413         unsigned int nr_pages;
 414
 415         pr_debug("swsusp: critical section: \n");
 416
 417         drain_local_pages();
 418         nr_pages = count_data_pages();
 419         printk("swsusp: Need to copy %u pages\n", nr_pages);
 420
 421         pr_debug("swsusp: pages needed: %u + %lu + %u, free: %u\n",
 422                  nr_pages,
 423                  (nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE,
 424                  PAGES_FOR_IO, nr_free_pages());
 425
 426         if (!enough_free_mem(nr_pages)) {
 427                 printk(KERN_ERR "swsusp: Not enough free memory\n");
 428                 return -ENOMEM;
 429         }
 430
 431         pagedir_nosave = swsusp_alloc(nr_pages);
 432         if (!pagedir_nosave)
 433                 return -ENOMEM;
 434
 435         /* During allocating of suspend pagedir, new cold pages may appear.
 436          * Kill them.
 437          */
 438         drain_local_pages();
 439         copy_data_pages(pagedir_nosave);
 440
 441         /*
 442          * End of critical section. From now on, we can write to memory,
 443          * but we should not touch disk. This specially means we must _not_
 444          * touch swap space! Except we must write out our image of course.
 445          */
 446
 447         nr_copy_pages = nr_pages;
 448
 449         printk("swsusp: critical section/: done (%d pages copied)\n", nr_pages);
 450         return 0;
 451 }