Merge remote-tracking branches 'regulator/topic/supply', 'regulator/topic/tps6105x...
[firefly-linux-kernel-4.4.55.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) Nadia Yvette Chambers, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/module.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/compiler.h>
17 #include <linux/cpuset.h>
18 #include <linux/mutex.h>
19 #include <linux/bootmem.h>
20 #include <linux/sysfs.h>
21 #include <linux/slab.h>
22 #include <linux/rmap.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
25 #include <linux/page-isolation.h>
26 #include <linux/jhash.h>
27
28 #include <asm/page.h>
29 #include <asm/pgtable.h>
30 #include <asm/tlb.h>
31
32 #include <linux/io.h>
33 #include <linux/hugetlb.h>
34 #include <linux/hugetlb_cgroup.h>
35 #include <linux/node.h>
36 #include "internal.h"
37
38 int hugepages_treat_as_movable;
39
40 int hugetlb_max_hstate __read_mostly;
41 unsigned int default_hstate_idx;
42 struct hstate hstates[HUGE_MAX_HSTATE];
43 /*
44  * Minimum page order among possible hugepage sizes, set to a proper value
45  * at boot time.
46  */
47 static unsigned int minimum_order __read_mostly = UINT_MAX;
48
49 __initdata LIST_HEAD(huge_boot_pages);
50
51 /* for command line parsing */
52 static struct hstate * __initdata parsed_hstate;
53 static unsigned long __initdata default_hstate_max_huge_pages;
54 static unsigned long __initdata default_hstate_size;
55
56 /*
57  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
58  * free_huge_pages, and surplus_huge_pages.
59  */
60 DEFINE_SPINLOCK(hugetlb_lock);
61
62 /*
63  * Serializes faults on the same logical page.  This is used to
64  * prevent spurious OOMs when the hugepage pool is fully utilized.
65  */
66 static int num_fault_mutexes;
67 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
68
69 /* Forward declaration */
70 static int hugetlb_acct_memory(struct hstate *h, long delta);
71
72 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
73 {
74         bool free = (spool->count == 0) && (spool->used_hpages == 0);
75
76         spin_unlock(&spool->lock);
77
78         /* If no pages are used, and no other handles to the subpool
79          * remain, give up any reservations mased on minimum size and
80          * free the subpool */
81         if (free) {
82                 if (spool->min_hpages != -1)
83                         hugetlb_acct_memory(spool->hstate,
84                                                 -spool->min_hpages);
85                 kfree(spool);
86         }
87 }
88
89 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
90                                                 long min_hpages)
91 {
92         struct hugepage_subpool *spool;
93
94         spool = kzalloc(sizeof(*spool), GFP_KERNEL);
95         if (!spool)
96                 return NULL;
97
98         spin_lock_init(&spool->lock);
99         spool->count = 1;
100         spool->max_hpages = max_hpages;
101         spool->hstate = h;
102         spool->min_hpages = min_hpages;
103
104         if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
105                 kfree(spool);
106                 return NULL;
107         }
108         spool->rsv_hpages = min_hpages;
109
110         return spool;
111 }
112
113 void hugepage_put_subpool(struct hugepage_subpool *spool)
114 {
115         spin_lock(&spool->lock);
116         BUG_ON(!spool->count);
117         spool->count--;
118         unlock_or_release_subpool(spool);
119 }
120
121 /*
122  * Subpool accounting for allocating and reserving pages.
123  * Return -ENOMEM if there are not enough resources to satisfy the
124  * the request.  Otherwise, return the number of pages by which the
125  * global pools must be adjusted (upward).  The returned value may
126  * only be different than the passed value (delta) in the case where
127  * a subpool minimum size must be manitained.
128  */
129 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
130                                       long delta)
131 {
132         long ret = delta;
133
134         if (!spool)
135                 return ret;
136
137         spin_lock(&spool->lock);
138
139         if (spool->max_hpages != -1) {          /* maximum size accounting */
140                 if ((spool->used_hpages + delta) <= spool->max_hpages)
141                         spool->used_hpages += delta;
142                 else {
143                         ret = -ENOMEM;
144                         goto unlock_ret;
145                 }
146         }
147
148         if (spool->min_hpages != -1) {          /* minimum size accounting */
149                 if (delta > spool->rsv_hpages) {
150                         /*
151                          * Asking for more reserves than those already taken on
152                          * behalf of subpool.  Return difference.
153                          */
154                         ret = delta - spool->rsv_hpages;
155                         spool->rsv_hpages = 0;
156                 } else {
157                         ret = 0;        /* reserves already accounted for */
158                         spool->rsv_hpages -= delta;
159                 }
160         }
161
162 unlock_ret:
163         spin_unlock(&spool->lock);
164         return ret;
165 }
166
167 /*
168  * Subpool accounting for freeing and unreserving pages.
169  * Return the number of global page reservations that must be dropped.
170  * The return value may only be different than the passed value (delta)
171  * in the case where a subpool minimum size must be maintained.
172  */
173 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
174                                        long delta)
175 {
176         long ret = delta;
177
178         if (!spool)
179                 return delta;
180
181         spin_lock(&spool->lock);
182
183         if (spool->max_hpages != -1)            /* maximum size accounting */
184                 spool->used_hpages -= delta;
185
186         if (spool->min_hpages != -1) {          /* minimum size accounting */
187                 if (spool->rsv_hpages + delta <= spool->min_hpages)
188                         ret = 0;
189                 else
190                         ret = spool->rsv_hpages + delta - spool->min_hpages;
191
192                 spool->rsv_hpages += delta;
193                 if (spool->rsv_hpages > spool->min_hpages)
194                         spool->rsv_hpages = spool->min_hpages;
195         }
196
197         /*
198          * If hugetlbfs_put_super couldn't free spool due to an outstanding
199          * quota reference, free it now.
200          */
201         unlock_or_release_subpool(spool);
202
203         return ret;
204 }
205
206 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
207 {
208         return HUGETLBFS_SB(inode->i_sb)->spool;
209 }
210
211 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
212 {
213         return subpool_inode(file_inode(vma->vm_file));
214 }
215
216 /*
217  * Region tracking -- allows tracking of reservations and instantiated pages
218  *                    across the pages in a mapping.
219  *
220  * The region data structures are embedded into a resv_map and protected
221  * by a resv_map's lock.  The set of regions within the resv_map represent
222  * reservations for huge pages, or huge pages that have already been
223  * instantiated within the map.  The from and to elements are huge page
224  * indicies into the associated mapping.  from indicates the starting index
225  * of the region.  to represents the first index past the end of  the region.
226  *
227  * For example, a file region structure with from == 0 and to == 4 represents
228  * four huge pages in a mapping.  It is important to note that the to element
229  * represents the first element past the end of the region. This is used in
230  * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
231  *
232  * Interval notation of the form [from, to) will be used to indicate that
233  * the endpoint from is inclusive and to is exclusive.
234  */
235 struct file_region {
236         struct list_head link;
237         long from;
238         long to;
239 };
240
241 /*
242  * Add the huge page range represented by [f, t) to the reserve
243  * map.  In the normal case, existing regions will be expanded
244  * to accommodate the specified range.  Sufficient regions should
245  * exist for expansion due to the previous call to region_chg
246  * with the same range.  However, it is possible that region_del
247  * could have been called after region_chg and modifed the map
248  * in such a way that no region exists to be expanded.  In this
249  * case, pull a region descriptor from the cache associated with
250  * the map and use that for the new range.
251  *
252  * Return the number of new huge pages added to the map.  This
253  * number is greater than or equal to zero.
254  */
255 static long region_add(struct resv_map *resv, long f, long t)
256 {
257         struct list_head *head = &resv->regions;
258         struct file_region *rg, *nrg, *trg;
259         long add = 0;
260
261         spin_lock(&resv->lock);
262         /* Locate the region we are either in or before. */
263         list_for_each_entry(rg, head, link)
264                 if (f <= rg->to)
265                         break;
266
267         /*
268          * If no region exists which can be expanded to include the
269          * specified range, the list must have been modified by an
270          * interleving call to region_del().  Pull a region descriptor
271          * from the cache and use it for this range.
272          */
273         if (&rg->link == head || t < rg->from) {
274                 VM_BUG_ON(resv->region_cache_count <= 0);
275
276                 resv->region_cache_count--;
277                 nrg = list_first_entry(&resv->region_cache, struct file_region,
278                                         link);
279                 list_del(&nrg->link);
280
281                 nrg->from = f;
282                 nrg->to = t;
283                 list_add(&nrg->link, rg->link.prev);
284
285                 add += t - f;
286                 goto out_locked;
287         }
288
289         /* Round our left edge to the current segment if it encloses us. */
290         if (f > rg->from)
291                 f = rg->from;
292
293         /* Check for and consume any regions we now overlap with. */
294         nrg = rg;
295         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
296                 if (&rg->link == head)
297                         break;
298                 if (rg->from > t)
299                         break;
300
301                 /* If this area reaches higher then extend our area to
302                  * include it completely.  If this is not the first area
303                  * which we intend to reuse, free it. */
304                 if (rg->to > t)
305                         t = rg->to;
306                 if (rg != nrg) {
307                         /* Decrement return value by the deleted range.
308                          * Another range will span this area so that by
309                          * end of routine add will be >= zero
310                          */
311                         add -= (rg->to - rg->from);
312                         list_del(&rg->link);
313                         kfree(rg);
314                 }
315         }
316
317         add += (nrg->from - f);         /* Added to beginning of region */
318         nrg->from = f;
319         add += t - nrg->to;             /* Added to end of region */
320         nrg->to = t;
321
322 out_locked:
323         resv->adds_in_progress--;
324         spin_unlock(&resv->lock);
325         VM_BUG_ON(add < 0);
326         return add;
327 }
328
329 /*
330  * Examine the existing reserve map and determine how many
331  * huge pages in the specified range [f, t) are NOT currently
332  * represented.  This routine is called before a subsequent
333  * call to region_add that will actually modify the reserve
334  * map to add the specified range [f, t).  region_chg does
335  * not change the number of huge pages represented by the
336  * map.  However, if the existing regions in the map can not
337  * be expanded to represent the new range, a new file_region
338  * structure is added to the map as a placeholder.  This is
339  * so that the subsequent region_add call will have all the
340  * regions it needs and will not fail.
341  *
342  * Upon entry, region_chg will also examine the cache of region descriptors
343  * associated with the map.  If there are not enough descriptors cached, one
344  * will be allocated for the in progress add operation.
345  *
346  * Returns the number of huge pages that need to be added to the existing
347  * reservation map for the range [f, t).  This number is greater or equal to
348  * zero.  -ENOMEM is returned if a new file_region structure or cache entry
349  * is needed and can not be allocated.
350  */
351 static long region_chg(struct resv_map *resv, long f, long t)
352 {
353         struct list_head *head = &resv->regions;
354         struct file_region *rg, *nrg = NULL;
355         long chg = 0;
356
357 retry:
358         spin_lock(&resv->lock);
359 retry_locked:
360         resv->adds_in_progress++;
361
362         /*
363          * Check for sufficient descriptors in the cache to accommodate
364          * the number of in progress add operations.
365          */
366         if (resv->adds_in_progress > resv->region_cache_count) {
367                 struct file_region *trg;
368
369                 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
370                 /* Must drop lock to allocate a new descriptor. */
371                 resv->adds_in_progress--;
372                 spin_unlock(&resv->lock);
373
374                 trg = kmalloc(sizeof(*trg), GFP_KERNEL);
375                 if (!trg)
376                         return -ENOMEM;
377
378                 spin_lock(&resv->lock);
379                 list_add(&trg->link, &resv->region_cache);
380                 resv->region_cache_count++;
381                 goto retry_locked;
382         }
383
384         /* Locate the region we are before or in. */
385         list_for_each_entry(rg, head, link)
386                 if (f <= rg->to)
387                         break;
388
389         /* If we are below the current region then a new region is required.
390          * Subtle, allocate a new region at the position but make it zero
391          * size such that we can guarantee to record the reservation. */
392         if (&rg->link == head || t < rg->from) {
393                 if (!nrg) {
394                         resv->adds_in_progress--;
395                         spin_unlock(&resv->lock);
396                         nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
397                         if (!nrg)
398                                 return -ENOMEM;
399
400                         nrg->from = f;
401                         nrg->to   = f;
402                         INIT_LIST_HEAD(&nrg->link);
403                         goto retry;
404                 }
405
406                 list_add(&nrg->link, rg->link.prev);
407                 chg = t - f;
408                 goto out_nrg;
409         }
410
411         /* Round our left edge to the current segment if it encloses us. */
412         if (f > rg->from)
413                 f = rg->from;
414         chg = t - f;
415
416         /* Check for and consume any regions we now overlap with. */
417         list_for_each_entry(rg, rg->link.prev, link) {
418                 if (&rg->link == head)
419                         break;
420                 if (rg->from > t)
421                         goto out;
422
423                 /* We overlap with this area, if it extends further than
424                  * us then we must extend ourselves.  Account for its
425                  * existing reservation. */
426                 if (rg->to > t) {
427                         chg += rg->to - t;
428                         t = rg->to;
429                 }
430                 chg -= rg->to - rg->from;
431         }
432
433 out:
434         spin_unlock(&resv->lock);
435         /*  We already know we raced and no longer need the new region */
436         kfree(nrg);
437         return chg;
438 out_nrg:
439         spin_unlock(&resv->lock);
440         return chg;
441 }
442
443 /*
444  * Abort the in progress add operation.  The adds_in_progress field
445  * of the resv_map keeps track of the operations in progress between
446  * calls to region_chg and region_add.  Operations are sometimes
447  * aborted after the call to region_chg.  In such cases, region_abort
448  * is called to decrement the adds_in_progress counter.
449  *
450  * NOTE: The range arguments [f, t) are not needed or used in this
451  * routine.  They are kept to make reading the calling code easier as
452  * arguments will match the associated region_chg call.
453  */
454 static void region_abort(struct resv_map *resv, long f, long t)
455 {
456         spin_lock(&resv->lock);
457         VM_BUG_ON(!resv->region_cache_count);
458         resv->adds_in_progress--;
459         spin_unlock(&resv->lock);
460 }
461
462 /*
463  * Delete the specified range [f, t) from the reserve map.  If the
464  * t parameter is LONG_MAX, this indicates that ALL regions after f
465  * should be deleted.  Locate the regions which intersect [f, t)
466  * and either trim, delete or split the existing regions.
467  *
468  * Returns the number of huge pages deleted from the reserve map.
469  * In the normal case, the return value is zero or more.  In the
470  * case where a region must be split, a new region descriptor must
471  * be allocated.  If the allocation fails, -ENOMEM will be returned.
472  * NOTE: If the parameter t == LONG_MAX, then we will never split
473  * a region and possibly return -ENOMEM.  Callers specifying
474  * t == LONG_MAX do not need to check for -ENOMEM error.
475  */
476 static long region_del(struct resv_map *resv, long f, long t)
477 {
478         struct list_head *head = &resv->regions;
479         struct file_region *rg, *trg;
480         struct file_region *nrg = NULL;
481         long del = 0;
482
483 retry:
484         spin_lock(&resv->lock);
485         list_for_each_entry_safe(rg, trg, head, link) {
486                 if (rg->to <= f)
487                         continue;
488                 if (rg->from >= t)
489                         break;
490
491                 if (f > rg->from && t < rg->to) { /* Must split region */
492                         /*
493                          * Check for an entry in the cache before dropping
494                          * lock and attempting allocation.
495                          */
496                         if (!nrg &&
497                             resv->region_cache_count > resv->adds_in_progress) {
498                                 nrg = list_first_entry(&resv->region_cache,
499                                                         struct file_region,
500                                                         link);
501                                 list_del(&nrg->link);
502                                 resv->region_cache_count--;
503                         }
504
505                         if (!nrg) {
506                                 spin_unlock(&resv->lock);
507                                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
508                                 if (!nrg)
509                                         return -ENOMEM;
510                                 goto retry;
511                         }
512
513                         del += t - f;
514
515                         /* New entry for end of split region */
516                         nrg->from = t;
517                         nrg->to = rg->to;
518                         INIT_LIST_HEAD(&nrg->link);
519
520                         /* Original entry is trimmed */
521                         rg->to = f;
522
523                         list_add(&nrg->link, &rg->link);
524                         nrg = NULL;
525                         break;
526                 }
527
528                 if (f <= rg->from && t >= rg->to) { /* Remove entire region */
529                         del += rg->to - rg->from;
530                         list_del(&rg->link);
531                         kfree(rg);
532                         continue;
533                 }
534
535                 if (f <= rg->from) {    /* Trim beginning of region */
536                         del += t - rg->from;
537                         rg->from = t;
538                 } else {                /* Trim end of region */
539                         del += rg->to - f;
540                         rg->to = f;
541                 }
542         }
543
544         spin_unlock(&resv->lock);
545         kfree(nrg);
546         return del;
547 }
548
549 /*
550  * A rare out of memory error was encountered which prevented removal of
551  * the reserve map region for a page.  The huge page itself was free'ed
552  * and removed from the page cache.  This routine will adjust the subpool
553  * usage count, and the global reserve count if needed.  By incrementing
554  * these counts, the reserve map entry which could not be deleted will
555  * appear as a "reserved" entry instead of simply dangling with incorrect
556  * counts.
557  */
558 void hugetlb_fix_reserve_counts(struct inode *inode, bool restore_reserve)
559 {
560         struct hugepage_subpool *spool = subpool_inode(inode);
561         long rsv_adjust;
562
563         rsv_adjust = hugepage_subpool_get_pages(spool, 1);
564         if (restore_reserve && rsv_adjust) {
565                 struct hstate *h = hstate_inode(inode);
566
567                 hugetlb_acct_memory(h, 1);
568         }
569 }
570
571 /*
572  * Count and return the number of huge pages in the reserve map
573  * that intersect with the range [f, t).
574  */
575 static long region_count(struct resv_map *resv, long f, long t)
576 {
577         struct list_head *head = &resv->regions;
578         struct file_region *rg;
579         long chg = 0;
580
581         spin_lock(&resv->lock);
582         /* Locate each segment we overlap with, and count that overlap. */
583         list_for_each_entry(rg, head, link) {
584                 long seg_from;
585                 long seg_to;
586
587                 if (rg->to <= f)
588                         continue;
589                 if (rg->from >= t)
590                         break;
591
592                 seg_from = max(rg->from, f);
593                 seg_to = min(rg->to, t);
594
595                 chg += seg_to - seg_from;
596         }
597         spin_unlock(&resv->lock);
598
599         return chg;
600 }
601
602 /*
603  * Convert the address within this vma to the page offset within
604  * the mapping, in pagecache page units; huge pages here.
605  */
606 static pgoff_t vma_hugecache_offset(struct hstate *h,
607                         struct vm_area_struct *vma, unsigned long address)
608 {
609         return ((address - vma->vm_start) >> huge_page_shift(h)) +
610                         (vma->vm_pgoff >> huge_page_order(h));
611 }
612
613 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
614                                      unsigned long address)
615 {
616         return vma_hugecache_offset(hstate_vma(vma), vma, address);
617 }
618
619 /*
620  * Return the size of the pages allocated when backing a VMA. In the majority
621  * cases this will be same size as used by the page table entries.
622  */
623 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
624 {
625         struct hstate *hstate;
626
627         if (!is_vm_hugetlb_page(vma))
628                 return PAGE_SIZE;
629
630         hstate = hstate_vma(vma);
631
632         return 1UL << huge_page_shift(hstate);
633 }
634 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
635
636 /*
637  * Return the page size being used by the MMU to back a VMA. In the majority
638  * of cases, the page size used by the kernel matches the MMU size. On
639  * architectures where it differs, an architecture-specific version of this
640  * function is required.
641  */
642 #ifndef vma_mmu_pagesize
643 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
644 {
645         return vma_kernel_pagesize(vma);
646 }
647 #endif
648
649 /*
650  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
651  * bits of the reservation map pointer, which are always clear due to
652  * alignment.
653  */
654 #define HPAGE_RESV_OWNER    (1UL << 0)
655 #define HPAGE_RESV_UNMAPPED (1UL << 1)
656 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
657
658 /*
659  * These helpers are used to track how many pages are reserved for
660  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
661  * is guaranteed to have their future faults succeed.
662  *
663  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
664  * the reserve counters are updated with the hugetlb_lock held. It is safe
665  * to reset the VMA at fork() time as it is not in use yet and there is no
666  * chance of the global counters getting corrupted as a result of the values.
667  *
668  * The private mapping reservation is represented in a subtly different
669  * manner to a shared mapping.  A shared mapping has a region map associated
670  * with the underlying file, this region map represents the backing file
671  * pages which have ever had a reservation assigned which this persists even
672  * after the page is instantiated.  A private mapping has a region map
673  * associated with the original mmap which is attached to all VMAs which
674  * reference it, this region map represents those offsets which have consumed
675  * reservation ie. where pages have been instantiated.
676  */
677 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
678 {
679         return (unsigned long)vma->vm_private_data;
680 }
681
682 static void set_vma_private_data(struct vm_area_struct *vma,
683                                                         unsigned long value)
684 {
685         vma->vm_private_data = (void *)value;
686 }
687
688 struct resv_map *resv_map_alloc(void)
689 {
690         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
691         struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
692
693         if (!resv_map || !rg) {
694                 kfree(resv_map);
695                 kfree(rg);
696                 return NULL;
697         }
698
699         kref_init(&resv_map->refs);
700         spin_lock_init(&resv_map->lock);
701         INIT_LIST_HEAD(&resv_map->regions);
702
703         resv_map->adds_in_progress = 0;
704
705         INIT_LIST_HEAD(&resv_map->region_cache);
706         list_add(&rg->link, &resv_map->region_cache);
707         resv_map->region_cache_count = 1;
708
709         return resv_map;
710 }
711
712 void resv_map_release(struct kref *ref)
713 {
714         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
715         struct list_head *head = &resv_map->region_cache;
716         struct file_region *rg, *trg;
717
718         /* Clear out any active regions before we release the map. */
719         region_del(resv_map, 0, LONG_MAX);
720
721         /* ... and any entries left in the cache */
722         list_for_each_entry_safe(rg, trg, head, link) {
723                 list_del(&rg->link);
724                 kfree(rg);
725         }
726
727         VM_BUG_ON(resv_map->adds_in_progress);
728
729         kfree(resv_map);
730 }
731
732 static inline struct resv_map *inode_resv_map(struct inode *inode)
733 {
734         return inode->i_mapping->private_data;
735 }
736
737 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
738 {
739         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
740         if (vma->vm_flags & VM_MAYSHARE) {
741                 struct address_space *mapping = vma->vm_file->f_mapping;
742                 struct inode *inode = mapping->host;
743
744                 return inode_resv_map(inode);
745
746         } else {
747                 return (struct resv_map *)(get_vma_private_data(vma) &
748                                                         ~HPAGE_RESV_MASK);
749         }
750 }
751
752 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
753 {
754         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
755         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
756
757         set_vma_private_data(vma, (get_vma_private_data(vma) &
758                                 HPAGE_RESV_MASK) | (unsigned long)map);
759 }
760
761 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
762 {
763         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
764         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
765
766         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
767 }
768
769 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
770 {
771         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
772
773         return (get_vma_private_data(vma) & flag) != 0;
774 }
775
776 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
777 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
778 {
779         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
780         if (!(vma->vm_flags & VM_MAYSHARE))
781                 vma->vm_private_data = (void *)0;
782 }
783
784 /* Returns true if the VMA has associated reserve pages */
785 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
786 {
787         if (vma->vm_flags & VM_NORESERVE) {
788                 /*
789                  * This address is already reserved by other process(chg == 0),
790                  * so, we should decrement reserved count. Without decrementing,
791                  * reserve count remains after releasing inode, because this
792                  * allocated page will go into page cache and is regarded as
793                  * coming from reserved pool in releasing step.  Currently, we
794                  * don't have any other solution to deal with this situation
795                  * properly, so add work-around here.
796                  */
797                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
798                         return true;
799                 else
800                         return false;
801         }
802
803         /* Shared mappings always use reserves */
804         if (vma->vm_flags & VM_MAYSHARE) {
805                 /*
806                  * We know VM_NORESERVE is not set.  Therefore, there SHOULD
807                  * be a region map for all pages.  The only situation where
808                  * there is no region map is if a hole was punched via
809                  * fallocate.  In this case, there really are no reverves to
810                  * use.  This situation is indicated if chg != 0.
811                  */
812                 if (chg)
813                         return false;
814                 else
815                         return true;
816         }
817
818         /*
819          * Only the process that called mmap() has reserves for
820          * private mappings.
821          */
822         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
823                 return true;
824
825         return false;
826 }
827
828 static void enqueue_huge_page(struct hstate *h, struct page *page)
829 {
830         int nid = page_to_nid(page);
831         list_move(&page->lru, &h->hugepage_freelists[nid]);
832         h->free_huge_pages++;
833         h->free_huge_pages_node[nid]++;
834 }
835
836 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
837 {
838         struct page *page;
839
840         list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
841                 if (!is_migrate_isolate_page(page))
842                         break;
843         /*
844          * if 'non-isolated free hugepage' not found on the list,
845          * the allocation fails.
846          */
847         if (&h->hugepage_freelists[nid] == &page->lru)
848                 return NULL;
849         list_move(&page->lru, &h->hugepage_activelist);
850         set_page_refcounted(page);
851         h->free_huge_pages--;
852         h->free_huge_pages_node[nid]--;
853         return page;
854 }
855
856 /* Movability of hugepages depends on migration support. */
857 static inline gfp_t htlb_alloc_mask(struct hstate *h)
858 {
859         if (hugepages_treat_as_movable || hugepage_migration_supported(h))
860                 return GFP_HIGHUSER_MOVABLE;
861         else
862                 return GFP_HIGHUSER;
863 }
864
865 static struct page *dequeue_huge_page_vma(struct hstate *h,
866                                 struct vm_area_struct *vma,
867                                 unsigned long address, int avoid_reserve,
868                                 long chg)
869 {
870         struct page *page = NULL;
871         struct mempolicy *mpol;
872         nodemask_t *nodemask;
873         struct zonelist *zonelist;
874         struct zone *zone;
875         struct zoneref *z;
876         unsigned int cpuset_mems_cookie;
877
878         /*
879          * A child process with MAP_PRIVATE mappings created by their parent
880          * have no page reserves. This check ensures that reservations are
881          * not "stolen". The child may still get SIGKILLed
882          */
883         if (!vma_has_reserves(vma, chg) &&
884                         h->free_huge_pages - h->resv_huge_pages == 0)
885                 goto err;
886
887         /* If reserves cannot be used, ensure enough pages are in the pool */
888         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
889                 goto err;
890
891 retry_cpuset:
892         cpuset_mems_cookie = read_mems_allowed_begin();
893         zonelist = huge_zonelist(vma, address,
894                                         htlb_alloc_mask(h), &mpol, &nodemask);
895
896         for_each_zone_zonelist_nodemask(zone, z, zonelist,
897                                                 MAX_NR_ZONES - 1, nodemask) {
898                 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
899                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
900                         if (page) {
901                                 if (avoid_reserve)
902                                         break;
903                                 if (!vma_has_reserves(vma, chg))
904                                         break;
905
906                                 SetPagePrivate(page);
907                                 h->resv_huge_pages--;
908                                 break;
909                         }
910                 }
911         }
912
913         mpol_cond_put(mpol);
914         if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
915                 goto retry_cpuset;
916         return page;
917
918 err:
919         return NULL;
920 }
921
922 /*
923  * common helper functions for hstate_next_node_to_{alloc|free}.
924  * We may have allocated or freed a huge page based on a different
925  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
926  * be outside of *nodes_allowed.  Ensure that we use an allowed
927  * node for alloc or free.
928  */
929 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
930 {
931         nid = next_node(nid, *nodes_allowed);
932         if (nid == MAX_NUMNODES)
933                 nid = first_node(*nodes_allowed);
934         VM_BUG_ON(nid >= MAX_NUMNODES);
935
936         return nid;
937 }
938
939 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
940 {
941         if (!node_isset(nid, *nodes_allowed))
942                 nid = next_node_allowed(nid, nodes_allowed);
943         return nid;
944 }
945
946 /*
947  * returns the previously saved node ["this node"] from which to
948  * allocate a persistent huge page for the pool and advance the
949  * next node from which to allocate, handling wrap at end of node
950  * mask.
951  */
952 static int hstate_next_node_to_alloc(struct hstate *h,
953                                         nodemask_t *nodes_allowed)
954 {
955         int nid;
956
957         VM_BUG_ON(!nodes_allowed);
958
959         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
960         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
961
962         return nid;
963 }
964
965 /*
966  * helper for free_pool_huge_page() - return the previously saved
967  * node ["this node"] from which to free a huge page.  Advance the
968  * next node id whether or not we find a free huge page to free so
969  * that the next attempt to free addresses the next node.
970  */
971 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
972 {
973         int nid;
974
975         VM_BUG_ON(!nodes_allowed);
976
977         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
978         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
979
980         return nid;
981 }
982
983 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
984         for (nr_nodes = nodes_weight(*mask);                            \
985                 nr_nodes > 0 &&                                         \
986                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
987                 nr_nodes--)
988
989 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
990         for (nr_nodes = nodes_weight(*mask);                            \
991                 nr_nodes > 0 &&                                         \
992                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
993                 nr_nodes--)
994
995 #if defined(CONFIG_CMA) && defined(CONFIG_X86_64)
996 static void destroy_compound_gigantic_page(struct page *page,
997                                         unsigned long order)
998 {
999         int i;
1000         int nr_pages = 1 << order;
1001         struct page *p = page + 1;
1002
1003         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1004                 __ClearPageTail(p);
1005                 set_page_refcounted(p);
1006                 p->first_page = NULL;
1007         }
1008
1009         set_compound_order(page, 0);
1010         __ClearPageHead(page);
1011 }
1012
1013 static void free_gigantic_page(struct page *page, unsigned order)
1014 {
1015         free_contig_range(page_to_pfn(page), 1 << order);
1016 }
1017
1018 static int __alloc_gigantic_page(unsigned long start_pfn,
1019                                 unsigned long nr_pages)
1020 {
1021         unsigned long end_pfn = start_pfn + nr_pages;
1022         return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1023 }
1024
1025 static bool pfn_range_valid_gigantic(unsigned long start_pfn,
1026                                 unsigned long nr_pages)
1027 {
1028         unsigned long i, end_pfn = start_pfn + nr_pages;
1029         struct page *page;
1030
1031         for (i = start_pfn; i < end_pfn; i++) {
1032                 if (!pfn_valid(i))
1033                         return false;
1034
1035                 page = pfn_to_page(i);
1036
1037                 if (PageReserved(page))
1038                         return false;
1039
1040                 if (page_count(page) > 0)
1041                         return false;
1042
1043                 if (PageHuge(page))
1044                         return false;
1045         }
1046
1047         return true;
1048 }
1049
1050 static bool zone_spans_last_pfn(const struct zone *zone,
1051                         unsigned long start_pfn, unsigned long nr_pages)
1052 {
1053         unsigned long last_pfn = start_pfn + nr_pages - 1;
1054         return zone_spans_pfn(zone, last_pfn);
1055 }
1056
1057 static struct page *alloc_gigantic_page(int nid, unsigned order)
1058 {
1059         unsigned long nr_pages = 1 << order;
1060         unsigned long ret, pfn, flags;
1061         struct zone *z;
1062
1063         z = NODE_DATA(nid)->node_zones;
1064         for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
1065                 spin_lock_irqsave(&z->lock, flags);
1066
1067                 pfn = ALIGN(z->zone_start_pfn, nr_pages);
1068                 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
1069                         if (pfn_range_valid_gigantic(pfn, nr_pages)) {
1070                                 /*
1071                                  * We release the zone lock here because
1072                                  * alloc_contig_range() will also lock the zone
1073                                  * at some point. If there's an allocation
1074                                  * spinning on this lock, it may win the race
1075                                  * and cause alloc_contig_range() to fail...
1076                                  */
1077                                 spin_unlock_irqrestore(&z->lock, flags);
1078                                 ret = __alloc_gigantic_page(pfn, nr_pages);
1079                                 if (!ret)
1080                                         return pfn_to_page(pfn);
1081                                 spin_lock_irqsave(&z->lock, flags);
1082                         }
1083                         pfn += nr_pages;
1084                 }
1085
1086                 spin_unlock_irqrestore(&z->lock, flags);
1087         }
1088
1089         return NULL;
1090 }
1091
1092 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1093 static void prep_compound_gigantic_page(struct page *page, unsigned long order);
1094
1095 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
1096 {
1097         struct page *page;
1098
1099         page = alloc_gigantic_page(nid, huge_page_order(h));
1100         if (page) {
1101                 prep_compound_gigantic_page(page, huge_page_order(h));
1102                 prep_new_huge_page(h, page, nid);
1103         }
1104
1105         return page;
1106 }
1107
1108 static int alloc_fresh_gigantic_page(struct hstate *h,
1109                                 nodemask_t *nodes_allowed)
1110 {
1111         struct page *page = NULL;
1112         int nr_nodes, node;
1113
1114         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1115                 page = alloc_fresh_gigantic_page_node(h, node);
1116                 if (page)
1117                         return 1;
1118         }
1119
1120         return 0;
1121 }
1122
1123 static inline bool gigantic_page_supported(void) { return true; }
1124 #else
1125 static inline bool gigantic_page_supported(void) { return false; }
1126 static inline void free_gigantic_page(struct page *page, unsigned order) { }
1127 static inline void destroy_compound_gigantic_page(struct page *page,
1128                                                 unsigned long order) { }
1129 static inline int alloc_fresh_gigantic_page(struct hstate *h,
1130                                         nodemask_t *nodes_allowed) { return 0; }
1131 #endif
1132
1133 static void update_and_free_page(struct hstate *h, struct page *page)
1134 {
1135         int i;
1136
1137         if (hstate_is_gigantic(h) && !gigantic_page_supported())
1138                 return;
1139
1140         h->nr_huge_pages--;
1141         h->nr_huge_pages_node[page_to_nid(page)]--;
1142         for (i = 0; i < pages_per_huge_page(h); i++) {
1143                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
1144                                 1 << PG_referenced | 1 << PG_dirty |
1145                                 1 << PG_active | 1 << PG_private |
1146                                 1 << PG_writeback);
1147         }
1148         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1149         set_compound_page_dtor(page, NULL);
1150         set_page_refcounted(page);
1151         if (hstate_is_gigantic(h)) {
1152                 destroy_compound_gigantic_page(page, huge_page_order(h));
1153                 free_gigantic_page(page, huge_page_order(h));
1154         } else {
1155                 __free_pages(page, huge_page_order(h));
1156         }
1157 }
1158
1159 struct hstate *size_to_hstate(unsigned long size)
1160 {
1161         struct hstate *h;
1162
1163         for_each_hstate(h) {
1164                 if (huge_page_size(h) == size)
1165                         return h;
1166         }
1167         return NULL;
1168 }
1169
1170 /*
1171  * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1172  * to hstate->hugepage_activelist.)
1173  *
1174  * This function can be called for tail pages, but never returns true for them.
1175  */
1176 bool page_huge_active(struct page *page)
1177 {
1178         VM_BUG_ON_PAGE(!PageHuge(page), page);
1179         return PageHead(page) && PagePrivate(&page[1]);
1180 }
1181
1182 /* never called for tail page */
1183 static void set_page_huge_active(struct page *page)
1184 {
1185         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1186         SetPagePrivate(&page[1]);
1187 }
1188
1189 static void clear_page_huge_active(struct page *page)
1190 {
1191         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1192         ClearPagePrivate(&page[1]);
1193 }
1194
1195 void free_huge_page(struct page *page)
1196 {
1197         /*
1198          * Can't pass hstate in here because it is called from the
1199          * compound page destructor.
1200          */
1201         struct hstate *h = page_hstate(page);
1202         int nid = page_to_nid(page);
1203         struct hugepage_subpool *spool =
1204                 (struct hugepage_subpool *)page_private(page);
1205         bool restore_reserve;
1206
1207         set_page_private(page, 0);
1208         page->mapping = NULL;
1209         BUG_ON(page_count(page));
1210         BUG_ON(page_mapcount(page));
1211         restore_reserve = PagePrivate(page);
1212         ClearPagePrivate(page);
1213
1214         /*
1215          * A return code of zero implies that the subpool will be under its
1216          * minimum size if the reservation is not restored after page is free.
1217          * Therefore, force restore_reserve operation.
1218          */
1219         if (hugepage_subpool_put_pages(spool, 1) == 0)
1220                 restore_reserve = true;
1221
1222         spin_lock(&hugetlb_lock);
1223         clear_page_huge_active(page);
1224         hugetlb_cgroup_uncharge_page(hstate_index(h),
1225                                      pages_per_huge_page(h), page);
1226         if (restore_reserve)
1227                 h->resv_huge_pages++;
1228
1229         if (h->surplus_huge_pages_node[nid]) {
1230                 /* remove the page from active list */
1231                 list_del(&page->lru);
1232                 update_and_free_page(h, page);
1233                 h->surplus_huge_pages--;
1234                 h->surplus_huge_pages_node[nid]--;
1235         } else {
1236                 arch_clear_hugepage_flags(page);
1237                 enqueue_huge_page(h, page);
1238         }
1239         spin_unlock(&hugetlb_lock);
1240 }
1241
1242 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1243 {
1244         INIT_LIST_HEAD(&page->lru);
1245         set_compound_page_dtor(page, free_huge_page);
1246         spin_lock(&hugetlb_lock);
1247         set_hugetlb_cgroup(page, NULL);
1248         h->nr_huge_pages++;
1249         h->nr_huge_pages_node[nid]++;
1250         spin_unlock(&hugetlb_lock);
1251         put_page(page); /* free it into the hugepage allocator */
1252 }
1253
1254 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
1255 {
1256         int i;
1257         int nr_pages = 1 << order;
1258         struct page *p = page + 1;
1259
1260         /* we rely on prep_new_huge_page to set the destructor */
1261         set_compound_order(page, order);
1262         __SetPageHead(page);
1263         __ClearPageReserved(page);
1264         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1265                 /*
1266                  * For gigantic hugepages allocated through bootmem at
1267                  * boot, it's safer to be consistent with the not-gigantic
1268                  * hugepages and clear the PG_reserved bit from all tail pages
1269                  * too.  Otherwse drivers using get_user_pages() to access tail
1270                  * pages may get the reference counting wrong if they see
1271                  * PG_reserved set on a tail page (despite the head page not
1272                  * having PG_reserved set).  Enforcing this consistency between
1273                  * head and tail pages allows drivers to optimize away a check
1274                  * on the head page when they need know if put_page() is needed
1275                  * after get_user_pages().
1276                  */
1277                 __ClearPageReserved(p);
1278                 set_page_count(p, 0);
1279                 p->first_page = page;
1280                 /* Make sure p->first_page is always valid for PageTail() */
1281                 smp_wmb();
1282                 __SetPageTail(p);
1283         }
1284 }
1285
1286 /*
1287  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1288  * transparent huge pages.  See the PageTransHuge() documentation for more
1289  * details.
1290  */
1291 int PageHuge(struct page *page)
1292 {
1293         if (!PageCompound(page))
1294                 return 0;
1295
1296         page = compound_head(page);
1297         return get_compound_page_dtor(page) == free_huge_page;
1298 }
1299 EXPORT_SYMBOL_GPL(PageHuge);
1300
1301 /*
1302  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1303  * normal or transparent huge pages.
1304  */
1305 int PageHeadHuge(struct page *page_head)
1306 {
1307         if (!PageHead(page_head))
1308                 return 0;
1309
1310         return get_compound_page_dtor(page_head) == free_huge_page;
1311 }
1312
1313 pgoff_t __basepage_index(struct page *page)
1314 {
1315         struct page *page_head = compound_head(page);
1316         pgoff_t index = page_index(page_head);
1317         unsigned long compound_idx;
1318
1319         if (!PageHuge(page_head))
1320                 return page_index(page);
1321
1322         if (compound_order(page_head) >= MAX_ORDER)
1323                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1324         else
1325                 compound_idx = page - page_head;
1326
1327         return (index << compound_order(page_head)) + compound_idx;
1328 }
1329
1330 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1331 {
1332         struct page *page;
1333
1334         page = __alloc_pages_node(nid,
1335                 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1336                                                 __GFP_REPEAT|__GFP_NOWARN,
1337                 huge_page_order(h));
1338         if (page) {
1339                 prep_new_huge_page(h, page, nid);
1340         }
1341
1342         return page;
1343 }
1344
1345 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1346 {
1347         struct page *page;
1348         int nr_nodes, node;
1349         int ret = 0;
1350
1351         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1352                 page = alloc_fresh_huge_page_node(h, node);
1353                 if (page) {
1354                         ret = 1;
1355                         break;
1356                 }
1357         }
1358
1359         if (ret)
1360                 count_vm_event(HTLB_BUDDY_PGALLOC);
1361         else
1362                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1363
1364         return ret;
1365 }
1366
1367 /*
1368  * Free huge page from pool from next node to free.
1369  * Attempt to keep persistent huge pages more or less
1370  * balanced over allowed nodes.
1371  * Called with hugetlb_lock locked.
1372  */
1373 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1374                                                          bool acct_surplus)
1375 {
1376         int nr_nodes, node;
1377         int ret = 0;
1378
1379         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1380                 /*
1381                  * If we're returning unused surplus pages, only examine
1382                  * nodes with surplus pages.
1383                  */
1384                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1385                     !list_empty(&h->hugepage_freelists[node])) {
1386                         struct page *page =
1387                                 list_entry(h->hugepage_freelists[node].next,
1388                                           struct page, lru);
1389                         list_del(&page->lru);
1390                         h->free_huge_pages--;
1391                         h->free_huge_pages_node[node]--;
1392                         if (acct_surplus) {
1393                                 h->surplus_huge_pages--;
1394                                 h->surplus_huge_pages_node[node]--;
1395                         }
1396                         update_and_free_page(h, page);
1397                         ret = 1;
1398                         break;
1399                 }
1400         }
1401
1402         return ret;
1403 }
1404
1405 /*
1406  * Dissolve a given free hugepage into free buddy pages. This function does
1407  * nothing for in-use (including surplus) hugepages.
1408  */
1409 static void dissolve_free_huge_page(struct page *page)
1410 {
1411         spin_lock(&hugetlb_lock);
1412         if (PageHuge(page) && !page_count(page)) {
1413                 struct hstate *h = page_hstate(page);
1414                 int nid = page_to_nid(page);
1415                 list_del(&page->lru);
1416                 h->free_huge_pages--;
1417                 h->free_huge_pages_node[nid]--;
1418                 update_and_free_page(h, page);
1419         }
1420         spin_unlock(&hugetlb_lock);
1421 }
1422
1423 /*
1424  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1425  * make specified memory blocks removable from the system.
1426  * Note that start_pfn should aligned with (minimum) hugepage size.
1427  */
1428 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1429 {
1430         unsigned long pfn;
1431
1432         if (!hugepages_supported())
1433                 return;
1434
1435         VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << minimum_order));
1436         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order)
1437                 dissolve_free_huge_page(pfn_to_page(pfn));
1438 }
1439
1440 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
1441 {
1442         struct page *page;
1443         unsigned int r_nid;
1444
1445         if (hstate_is_gigantic(h))
1446                 return NULL;
1447
1448         /*
1449          * Assume we will successfully allocate the surplus page to
1450          * prevent racing processes from causing the surplus to exceed
1451          * overcommit
1452          *
1453          * This however introduces a different race, where a process B
1454          * tries to grow the static hugepage pool while alloc_pages() is
1455          * called by process A. B will only examine the per-node
1456          * counters in determining if surplus huge pages can be
1457          * converted to normal huge pages in adjust_pool_surplus(). A
1458          * won't be able to increment the per-node counter, until the
1459          * lock is dropped by B, but B doesn't drop hugetlb_lock until
1460          * no more huge pages can be converted from surplus to normal
1461          * state (and doesn't try to convert again). Thus, we have a
1462          * case where a surplus huge page exists, the pool is grown, and
1463          * the surplus huge page still exists after, even though it
1464          * should just have been converted to a normal huge page. This
1465          * does not leak memory, though, as the hugepage will be freed
1466          * once it is out of use. It also does not allow the counters to
1467          * go out of whack in adjust_pool_surplus() as we don't modify
1468          * the node values until we've gotten the hugepage and only the
1469          * per-node value is checked there.
1470          */
1471         spin_lock(&hugetlb_lock);
1472         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1473                 spin_unlock(&hugetlb_lock);
1474                 return NULL;
1475         } else {
1476                 h->nr_huge_pages++;
1477                 h->surplus_huge_pages++;
1478         }
1479         spin_unlock(&hugetlb_lock);
1480
1481         if (nid == NUMA_NO_NODE)
1482                 page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|
1483                                    __GFP_REPEAT|__GFP_NOWARN,
1484                                    huge_page_order(h));
1485         else
1486                 page = __alloc_pages_node(nid,
1487                         htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1488                         __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
1489
1490         spin_lock(&hugetlb_lock);
1491         if (page) {
1492                 INIT_LIST_HEAD(&page->lru);
1493                 r_nid = page_to_nid(page);
1494                 set_compound_page_dtor(page, free_huge_page);
1495                 set_hugetlb_cgroup(page, NULL);
1496                 /*
1497                  * We incremented the global counters already
1498                  */
1499                 h->nr_huge_pages_node[r_nid]++;
1500                 h->surplus_huge_pages_node[r_nid]++;
1501                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1502         } else {
1503                 h->nr_huge_pages--;
1504                 h->surplus_huge_pages--;
1505                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1506         }
1507         spin_unlock(&hugetlb_lock);
1508
1509         return page;
1510 }
1511
1512 /*
1513  * This allocation function is useful in the context where vma is irrelevant.
1514  * E.g. soft-offlining uses this function because it only cares physical
1515  * address of error page.
1516  */
1517 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1518 {
1519         struct page *page = NULL;
1520
1521         spin_lock(&hugetlb_lock);
1522         if (h->free_huge_pages - h->resv_huge_pages > 0)
1523                 page = dequeue_huge_page_node(h, nid);
1524         spin_unlock(&hugetlb_lock);
1525
1526         if (!page)
1527                 page = alloc_buddy_huge_page(h, nid);
1528
1529         return page;
1530 }
1531
1532 /*
1533  * Increase the hugetlb pool such that it can accommodate a reservation
1534  * of size 'delta'.
1535  */
1536 static int gather_surplus_pages(struct hstate *h, int delta)
1537 {
1538         struct list_head surplus_list;
1539         struct page *page, *tmp;
1540         int ret, i;
1541         int needed, allocated;
1542         bool alloc_ok = true;
1543
1544         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1545         if (needed <= 0) {
1546                 h->resv_huge_pages += delta;
1547                 return 0;
1548         }
1549
1550         allocated = 0;
1551         INIT_LIST_HEAD(&surplus_list);
1552
1553         ret = -ENOMEM;
1554 retry:
1555         spin_unlock(&hugetlb_lock);
1556         for (i = 0; i < needed; i++) {
1557                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1558                 if (!page) {
1559                         alloc_ok = false;
1560                         break;
1561                 }
1562                 list_add(&page->lru, &surplus_list);
1563         }
1564         allocated += i;
1565
1566         /*
1567          * After retaking hugetlb_lock, we need to recalculate 'needed'
1568          * because either resv_huge_pages or free_huge_pages may have changed.
1569          */
1570         spin_lock(&hugetlb_lock);
1571         needed = (h->resv_huge_pages + delta) -
1572                         (h->free_huge_pages + allocated);
1573         if (needed > 0) {
1574                 if (alloc_ok)
1575                         goto retry;
1576                 /*
1577                  * We were not able to allocate enough pages to
1578                  * satisfy the entire reservation so we free what
1579                  * we've allocated so far.
1580                  */
1581                 goto free;
1582         }
1583         /*
1584          * The surplus_list now contains _at_least_ the number of extra pages
1585          * needed to accommodate the reservation.  Add the appropriate number
1586          * of pages to the hugetlb pool and free the extras back to the buddy
1587          * allocator.  Commit the entire reservation here to prevent another
1588          * process from stealing the pages as they are added to the pool but
1589          * before they are reserved.
1590          */
1591         needed += allocated;
1592         h->resv_huge_pages += delta;
1593         ret = 0;
1594
1595         /* Free the needed pages to the hugetlb pool */
1596         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1597                 if ((--needed) < 0)
1598                         break;
1599                 /*
1600                  * This page is now managed by the hugetlb allocator and has
1601                  * no users -- drop the buddy allocator's reference.
1602                  */
1603                 put_page_testzero(page);
1604                 VM_BUG_ON_PAGE(page_count(page), page);
1605                 enqueue_huge_page(h, page);
1606         }
1607 free:
1608         spin_unlock(&hugetlb_lock);
1609
1610         /* Free unnecessary surplus pages to the buddy allocator */
1611         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1612                 put_page(page);
1613         spin_lock(&hugetlb_lock);
1614
1615         return ret;
1616 }
1617
1618 /*
1619  * When releasing a hugetlb pool reservation, any surplus pages that were
1620  * allocated to satisfy the reservation must be explicitly freed if they were
1621  * never used.
1622  * Called with hugetlb_lock held.
1623  */
1624 static void return_unused_surplus_pages(struct hstate *h,
1625                                         unsigned long unused_resv_pages)
1626 {
1627         unsigned long nr_pages;
1628
1629         /* Uncommit the reservation */
1630         h->resv_huge_pages -= unused_resv_pages;
1631
1632         /* Cannot return gigantic pages currently */
1633         if (hstate_is_gigantic(h))
1634                 return;
1635
1636         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1637
1638         /*
1639          * We want to release as many surplus pages as possible, spread
1640          * evenly across all nodes with memory. Iterate across these nodes
1641          * until we can no longer free unreserved surplus pages. This occurs
1642          * when the nodes with surplus pages have no free pages.
1643          * free_pool_huge_page() will balance the the freed pages across the
1644          * on-line nodes with memory and will handle the hstate accounting.
1645          */
1646         while (nr_pages--) {
1647                 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1648                         break;
1649                 cond_resched_lock(&hugetlb_lock);
1650         }
1651 }
1652
1653
1654 /*
1655  * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
1656  * are used by the huge page allocation routines to manage reservations.
1657  *
1658  * vma_needs_reservation is called to determine if the huge page at addr
1659  * within the vma has an associated reservation.  If a reservation is
1660  * needed, the value 1 is returned.  The caller is then responsible for
1661  * managing the global reservation and subpool usage counts.  After
1662  * the huge page has been allocated, vma_commit_reservation is called
1663  * to add the page to the reservation map.  If the page allocation fails,
1664  * the reservation must be ended instead of committed.  vma_end_reservation
1665  * is called in such cases.
1666  *
1667  * In the normal case, vma_commit_reservation returns the same value
1668  * as the preceding vma_needs_reservation call.  The only time this
1669  * is not the case is if a reserve map was changed between calls.  It
1670  * is the responsibility of the caller to notice the difference and
1671  * take appropriate action.
1672  */
1673 enum vma_resv_mode {
1674         VMA_NEEDS_RESV,
1675         VMA_COMMIT_RESV,
1676         VMA_END_RESV,
1677 };
1678 static long __vma_reservation_common(struct hstate *h,
1679                                 struct vm_area_struct *vma, unsigned long addr,
1680                                 enum vma_resv_mode mode)
1681 {
1682         struct resv_map *resv;
1683         pgoff_t idx;
1684         long ret;
1685
1686         resv = vma_resv_map(vma);
1687         if (!resv)
1688                 return 1;
1689
1690         idx = vma_hugecache_offset(h, vma, addr);
1691         switch (mode) {
1692         case VMA_NEEDS_RESV:
1693                 ret = region_chg(resv, idx, idx + 1);
1694                 break;
1695         case VMA_COMMIT_RESV:
1696                 ret = region_add(resv, idx, idx + 1);
1697                 break;
1698         case VMA_END_RESV:
1699                 region_abort(resv, idx, idx + 1);
1700                 ret = 0;
1701                 break;
1702         default:
1703                 BUG();
1704         }
1705
1706         if (vma->vm_flags & VM_MAYSHARE)
1707                 return ret;
1708         else
1709                 return ret < 0 ? ret : 0;
1710 }
1711
1712 static long vma_needs_reservation(struct hstate *h,
1713                         struct vm_area_struct *vma, unsigned long addr)
1714 {
1715         return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
1716 }
1717
1718 static long vma_commit_reservation(struct hstate *h,
1719                         struct vm_area_struct *vma, unsigned long addr)
1720 {
1721         return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
1722 }
1723
1724 static void vma_end_reservation(struct hstate *h,
1725                         struct vm_area_struct *vma, unsigned long addr)
1726 {
1727         (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
1728 }
1729
1730 struct page *alloc_huge_page(struct vm_area_struct *vma,
1731                                     unsigned long addr, int avoid_reserve)
1732 {
1733         struct hugepage_subpool *spool = subpool_vma(vma);
1734         struct hstate *h = hstate_vma(vma);
1735         struct page *page;
1736         long map_chg, map_commit;
1737         long gbl_chg;
1738         int ret, idx;
1739         struct hugetlb_cgroup *h_cg;
1740
1741         idx = hstate_index(h);
1742         /*
1743          * Examine the region/reserve map to determine if the process
1744          * has a reservation for the page to be allocated.  A return
1745          * code of zero indicates a reservation exists (no change).
1746          */
1747         map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
1748         if (map_chg < 0)
1749                 return ERR_PTR(-ENOMEM);
1750
1751         /*
1752          * Processes that did not create the mapping will have no
1753          * reserves as indicated by the region/reserve map. Check
1754          * that the allocation will not exceed the subpool limit.
1755          * Allocations for MAP_NORESERVE mappings also need to be
1756          * checked against any subpool limit.
1757          */
1758         if (map_chg || avoid_reserve) {
1759                 gbl_chg = hugepage_subpool_get_pages(spool, 1);
1760                 if (gbl_chg < 0) {
1761                         vma_end_reservation(h, vma, addr);
1762                         return ERR_PTR(-ENOSPC);
1763                 }
1764
1765                 /*
1766                  * Even though there was no reservation in the region/reserve
1767                  * map, there could be reservations associated with the
1768                  * subpool that can be used.  This would be indicated if the
1769                  * return value of hugepage_subpool_get_pages() is zero.
1770                  * However, if avoid_reserve is specified we still avoid even
1771                  * the subpool reservations.
1772                  */
1773                 if (avoid_reserve)
1774                         gbl_chg = 1;
1775         }
1776
1777         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1778         if (ret)
1779                 goto out_subpool_put;
1780
1781         spin_lock(&hugetlb_lock);
1782         /*
1783          * glb_chg is passed to indicate whether or not a page must be taken
1784          * from the global free pool (global change).  gbl_chg == 0 indicates
1785          * a reservation exists for the allocation.
1786          */
1787         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
1788         if (!page) {
1789                 spin_unlock(&hugetlb_lock);
1790                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1791                 if (!page)
1792                         goto out_uncharge_cgroup;
1793
1794                 spin_lock(&hugetlb_lock);
1795                 list_move(&page->lru, &h->hugepage_activelist);
1796                 /* Fall through */
1797         }
1798         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1799         spin_unlock(&hugetlb_lock);
1800
1801         set_page_private(page, (unsigned long)spool);
1802
1803         map_commit = vma_commit_reservation(h, vma, addr);
1804         if (unlikely(map_chg > map_commit)) {
1805                 /*
1806                  * The page was added to the reservation map between
1807                  * vma_needs_reservation and vma_commit_reservation.
1808                  * This indicates a race with hugetlb_reserve_pages.
1809                  * Adjust for the subpool count incremented above AND
1810                  * in hugetlb_reserve_pages for the same page.  Also,
1811                  * the reservation count added in hugetlb_reserve_pages
1812                  * no longer applies.
1813                  */
1814                 long rsv_adjust;
1815
1816                 rsv_adjust = hugepage_subpool_put_pages(spool, 1);
1817                 hugetlb_acct_memory(h, -rsv_adjust);
1818         }
1819         return page;
1820
1821 out_uncharge_cgroup:
1822         hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
1823 out_subpool_put:
1824         if (map_chg || avoid_reserve)
1825                 hugepage_subpool_put_pages(spool, 1);
1826         vma_end_reservation(h, vma, addr);
1827         return ERR_PTR(-ENOSPC);
1828 }
1829
1830 /*
1831  * alloc_huge_page()'s wrapper which simply returns the page if allocation
1832  * succeeds, otherwise NULL. This function is called from new_vma_page(),
1833  * where no ERR_VALUE is expected to be returned.
1834  */
1835 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1836                                 unsigned long addr, int avoid_reserve)
1837 {
1838         struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1839         if (IS_ERR(page))
1840                 page = NULL;
1841         return page;
1842 }
1843
1844 int __weak alloc_bootmem_huge_page(struct hstate *h)
1845 {
1846         struct huge_bootmem_page *m;
1847         int nr_nodes, node;
1848
1849         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
1850                 void *addr;
1851
1852                 addr = memblock_virt_alloc_try_nid_nopanic(
1853                                 huge_page_size(h), huge_page_size(h),
1854                                 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
1855                 if (addr) {
1856                         /*
1857                          * Use the beginning of the huge page to store the
1858                          * huge_bootmem_page struct (until gather_bootmem
1859                          * puts them into the mem_map).
1860                          */
1861                         m = addr;
1862                         goto found;
1863                 }
1864         }
1865         return 0;
1866
1867 found:
1868         BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
1869         /* Put them into a private list first because mem_map is not up yet */
1870         list_add(&m->list, &huge_boot_pages);
1871         m->hstate = h;
1872         return 1;
1873 }
1874
1875 static void __init prep_compound_huge_page(struct page *page, int order)
1876 {
1877         if (unlikely(order > (MAX_ORDER - 1)))
1878                 prep_compound_gigantic_page(page, order);
1879         else
1880                 prep_compound_page(page, order);
1881 }
1882
1883 /* Put bootmem huge pages into the standard lists after mem_map is up */
1884 static void __init gather_bootmem_prealloc(void)
1885 {
1886         struct huge_bootmem_page *m;
1887
1888         list_for_each_entry(m, &huge_boot_pages, list) {
1889                 struct hstate *h = m->hstate;
1890                 struct page *page;
1891
1892 #ifdef CONFIG_HIGHMEM
1893                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1894                 memblock_free_late(__pa(m),
1895                                    sizeof(struct huge_bootmem_page));
1896 #else
1897                 page = virt_to_page(m);
1898 #endif
1899                 WARN_ON(page_count(page) != 1);
1900                 prep_compound_huge_page(page, h->order);
1901                 WARN_ON(PageReserved(page));
1902                 prep_new_huge_page(h, page, page_to_nid(page));
1903                 /*
1904                  * If we had gigantic hugepages allocated at boot time, we need
1905                  * to restore the 'stolen' pages to totalram_pages in order to
1906                  * fix confusing memory reports from free(1) and another
1907                  * side-effects, like CommitLimit going negative.
1908                  */
1909                 if (hstate_is_gigantic(h))
1910                         adjust_managed_page_count(page, 1 << h->order);
1911         }
1912 }
1913
1914 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1915 {
1916         unsigned long i;
1917
1918         for (i = 0; i < h->max_huge_pages; ++i) {
1919                 if (hstate_is_gigantic(h)) {
1920                         if (!alloc_bootmem_huge_page(h))
1921                                 break;
1922                 } else if (!alloc_fresh_huge_page(h,
1923                                          &node_states[N_MEMORY]))
1924                         break;
1925         }
1926         h->max_huge_pages = i;
1927 }
1928
1929 static void __init hugetlb_init_hstates(void)
1930 {
1931         struct hstate *h;
1932
1933         for_each_hstate(h) {
1934                 if (minimum_order > huge_page_order(h))
1935                         minimum_order = huge_page_order(h);
1936
1937                 /* oversize hugepages were init'ed in early boot */
1938                 if (!hstate_is_gigantic(h))
1939                         hugetlb_hstate_alloc_pages(h);
1940         }
1941         VM_BUG_ON(minimum_order == UINT_MAX);
1942 }
1943
1944 static char * __init memfmt(char *buf, unsigned long n)
1945 {
1946         if (n >= (1UL << 30))
1947                 sprintf(buf, "%lu GB", n >> 30);
1948         else if (n >= (1UL << 20))
1949                 sprintf(buf, "%lu MB", n >> 20);
1950         else
1951                 sprintf(buf, "%lu KB", n >> 10);
1952         return buf;
1953 }
1954
1955 static void __init report_hugepages(void)
1956 {
1957         struct hstate *h;
1958
1959         for_each_hstate(h) {
1960                 char buf[32];
1961                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
1962                         memfmt(buf, huge_page_size(h)),
1963                         h->free_huge_pages);
1964         }
1965 }
1966
1967 #ifdef CONFIG_HIGHMEM
1968 static void try_to_free_low(struct hstate *h, unsigned long count,
1969                                                 nodemask_t *nodes_allowed)
1970 {
1971         int i;
1972
1973         if (hstate_is_gigantic(h))
1974                 return;
1975
1976         for_each_node_mask(i, *nodes_allowed) {
1977                 struct page *page, *next;
1978                 struct list_head *freel = &h->hugepage_freelists[i];
1979                 list_for_each_entry_safe(page, next, freel, lru) {
1980                         if (count >= h->nr_huge_pages)
1981                                 return;
1982                         if (PageHighMem(page))
1983                                 continue;
1984                         list_del(&page->lru);
1985                         update_and_free_page(h, page);
1986                         h->free_huge_pages--;
1987                         h->free_huge_pages_node[page_to_nid(page)]--;
1988                 }
1989         }
1990 }
1991 #else
1992 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1993                                                 nodemask_t *nodes_allowed)
1994 {
1995 }
1996 #endif
1997
1998 /*
1999  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
2000  * balanced by operating on them in a round-robin fashion.
2001  * Returns 1 if an adjustment was made.
2002  */
2003 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2004                                 int delta)
2005 {
2006         int nr_nodes, node;
2007
2008         VM_BUG_ON(delta != -1 && delta != 1);
2009
2010         if (delta < 0) {
2011                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2012                         if (h->surplus_huge_pages_node[node])
2013                                 goto found;
2014                 }
2015         } else {
2016                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2017                         if (h->surplus_huge_pages_node[node] <
2018                                         h->nr_huge_pages_node[node])
2019                                 goto found;
2020                 }
2021         }
2022         return 0;
2023
2024 found:
2025         h->surplus_huge_pages += delta;
2026         h->surplus_huge_pages_node[node] += delta;
2027         return 1;
2028 }
2029
2030 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2031 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
2032                                                 nodemask_t *nodes_allowed)
2033 {
2034         unsigned long min_count, ret;
2035
2036         if (hstate_is_gigantic(h) && !gigantic_page_supported())
2037                 return h->max_huge_pages;
2038
2039         /*
2040          * Increase the pool size
2041          * First take pages out of surplus state.  Then make up the
2042          * remaining difference by allocating fresh huge pages.
2043          *
2044          * We might race with alloc_buddy_huge_page() here and be unable
2045          * to convert a surplus huge page to a normal huge page. That is
2046          * not critical, though, it just means the overall size of the
2047          * pool might be one hugepage larger than it needs to be, but
2048          * within all the constraints specified by the sysctls.
2049          */
2050         spin_lock(&hugetlb_lock);
2051         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2052                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
2053                         break;
2054         }
2055
2056         while (count > persistent_huge_pages(h)) {
2057                 /*
2058                  * If this allocation races such that we no longer need the
2059                  * page, free_huge_page will handle it by freeing the page
2060                  * and reducing the surplus.
2061                  */
2062                 spin_unlock(&hugetlb_lock);
2063                 if (hstate_is_gigantic(h))
2064                         ret = alloc_fresh_gigantic_page(h, nodes_allowed);
2065                 else
2066                         ret = alloc_fresh_huge_page(h, nodes_allowed);
2067                 spin_lock(&hugetlb_lock);
2068                 if (!ret)
2069                         goto out;
2070
2071                 /* Bail for signals. Probably ctrl-c from user */
2072                 if (signal_pending(current))
2073                         goto out;
2074         }
2075
2076         /*
2077          * Decrease the pool size
2078          * First return free pages to the buddy allocator (being careful
2079          * to keep enough around to satisfy reservations).  Then place
2080          * pages into surplus state as needed so the pool will shrink
2081          * to the desired size as pages become free.
2082          *
2083          * By placing pages into the surplus state independent of the
2084          * overcommit value, we are allowing the surplus pool size to
2085          * exceed overcommit. There are few sane options here. Since
2086          * alloc_buddy_huge_page() is checking the global counter,
2087          * though, we'll note that we're not allowed to exceed surplus
2088          * and won't grow the pool anywhere else. Not until one of the
2089          * sysctls are changed, or the surplus pages go out of use.
2090          */
2091         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2092         min_count = max(count, min_count);
2093         try_to_free_low(h, min_count, nodes_allowed);
2094         while (min_count < persistent_huge_pages(h)) {
2095                 if (!free_pool_huge_page(h, nodes_allowed, 0))
2096                         break;
2097                 cond_resched_lock(&hugetlb_lock);
2098         }
2099         while (count < persistent_huge_pages(h)) {
2100                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
2101                         break;
2102         }
2103 out:
2104         ret = persistent_huge_pages(h);
2105         spin_unlock(&hugetlb_lock);
2106         return ret;
2107 }
2108
2109 #define HSTATE_ATTR_RO(_name) \
2110         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2111
2112 #define HSTATE_ATTR(_name) \
2113         static struct kobj_attribute _name##_attr = \
2114                 __ATTR(_name, 0644, _name##_show, _name##_store)
2115
2116 static struct kobject *hugepages_kobj;
2117 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2118
2119 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2120
2121 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2122 {
2123         int i;
2124
2125         for (i = 0; i < HUGE_MAX_HSTATE; i++)
2126                 if (hstate_kobjs[i] == kobj) {
2127                         if (nidp)
2128                                 *nidp = NUMA_NO_NODE;
2129                         return &hstates[i];
2130                 }
2131
2132         return kobj_to_node_hstate(kobj, nidp);
2133 }
2134
2135 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2136                                         struct kobj_attribute *attr, char *buf)
2137 {
2138         struct hstate *h;
2139         unsigned long nr_huge_pages;
2140         int nid;
2141
2142         h = kobj_to_hstate(kobj, &nid);
2143         if (nid == NUMA_NO_NODE)
2144                 nr_huge_pages = h->nr_huge_pages;
2145         else
2146                 nr_huge_pages = h->nr_huge_pages_node[nid];
2147
2148         return sprintf(buf, "%lu\n", nr_huge_pages);
2149 }
2150
2151 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2152                                            struct hstate *h, int nid,
2153                                            unsigned long count, size_t len)
2154 {
2155         int err;
2156         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
2157
2158         if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
2159                 err = -EINVAL;
2160                 goto out;
2161         }
2162
2163         if (nid == NUMA_NO_NODE) {
2164                 /*
2165                  * global hstate attribute
2166                  */
2167                 if (!(obey_mempolicy &&
2168                                 init_nodemask_of_mempolicy(nodes_allowed))) {
2169                         NODEMASK_FREE(nodes_allowed);
2170                         nodes_allowed = &node_states[N_MEMORY];
2171                 }
2172         } else if (nodes_allowed) {
2173                 /*
2174                  * per node hstate attribute: adjust count to global,
2175                  * but restrict alloc/free to the specified node.
2176                  */
2177                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2178                 init_nodemask_of_node(nodes_allowed, nid);
2179         } else
2180                 nodes_allowed = &node_states[N_MEMORY];
2181
2182         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
2183
2184         if (nodes_allowed != &node_states[N_MEMORY])
2185                 NODEMASK_FREE(nodes_allowed);
2186
2187         return len;
2188 out:
2189         NODEMASK_FREE(nodes_allowed);
2190         return err;
2191 }
2192
2193 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2194                                          struct kobject *kobj, const char *buf,
2195                                          size_t len)
2196 {
2197         struct hstate *h;
2198         unsigned long count;
2199         int nid;
2200         int err;
2201
2202         err = kstrtoul(buf, 10, &count);
2203         if (err)
2204                 return err;
2205
2206         h = kobj_to_hstate(kobj, &nid);
2207         return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2208 }
2209
2210 static ssize_t nr_hugepages_show(struct kobject *kobj,
2211                                        struct kobj_attribute *attr, char *buf)
2212 {
2213         return nr_hugepages_show_common(kobj, attr, buf);
2214 }
2215
2216 static ssize_t nr_hugepages_store(struct kobject *kobj,
2217                struct kobj_attribute *attr, const char *buf, size_t len)
2218 {
2219         return nr_hugepages_store_common(false, kobj, buf, len);
2220 }
2221 HSTATE_ATTR(nr_hugepages);
2222
2223 #ifdef CONFIG_NUMA
2224
2225 /*
2226  * hstate attribute for optionally mempolicy-based constraint on persistent
2227  * huge page alloc/free.
2228  */
2229 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
2230                                        struct kobj_attribute *attr, char *buf)
2231 {
2232         return nr_hugepages_show_common(kobj, attr, buf);
2233 }
2234
2235 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
2236                struct kobj_attribute *attr, const char *buf, size_t len)
2237 {
2238         return nr_hugepages_store_common(true, kobj, buf, len);
2239 }
2240 HSTATE_ATTR(nr_hugepages_mempolicy);
2241 #endif
2242
2243
2244 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
2245                                         struct kobj_attribute *attr, char *buf)
2246 {
2247         struct hstate *h = kobj_to_hstate(kobj, NULL);
2248         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
2249 }
2250
2251 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
2252                 struct kobj_attribute *attr, const char *buf, size_t count)
2253 {
2254         int err;
2255         unsigned long input;
2256         struct hstate *h = kobj_to_hstate(kobj, NULL);
2257
2258         if (hstate_is_gigantic(h))
2259                 return -EINVAL;
2260
2261         err = kstrtoul(buf, 10, &input);
2262         if (err)
2263                 return err;
2264
2265         spin_lock(&hugetlb_lock);
2266         h->nr_overcommit_huge_pages = input;
2267         spin_unlock(&hugetlb_lock);
2268
2269         return count;
2270 }
2271 HSTATE_ATTR(nr_overcommit_hugepages);
2272
2273 static ssize_t free_hugepages_show(struct kobject *kobj,
2274                                         struct kobj_attribute *attr, char *buf)
2275 {
2276         struct hstate *h;
2277         unsigned long free_huge_pages;
2278         int nid;
2279
2280         h = kobj_to_hstate(kobj, &nid);
2281         if (nid == NUMA_NO_NODE)
2282                 free_huge_pages = h->free_huge_pages;
2283         else
2284                 free_huge_pages = h->free_huge_pages_node[nid];
2285
2286         return sprintf(buf, "%lu\n", free_huge_pages);
2287 }
2288 HSTATE_ATTR_RO(free_hugepages);
2289
2290 static ssize_t resv_hugepages_show(struct kobject *kobj,
2291                                         struct kobj_attribute *attr, char *buf)
2292 {
2293         struct hstate *h = kobj_to_hstate(kobj, NULL);
2294         return sprintf(buf, "%lu\n", h->resv_huge_pages);
2295 }
2296 HSTATE_ATTR_RO(resv_hugepages);
2297
2298 static ssize_t surplus_hugepages_show(struct kobject *kobj,
2299                                         struct kobj_attribute *attr, char *buf)
2300 {
2301         struct hstate *h;
2302         unsigned long surplus_huge_pages;
2303         int nid;
2304
2305         h = kobj_to_hstate(kobj, &nid);
2306         if (nid == NUMA_NO_NODE)
2307                 surplus_huge_pages = h->surplus_huge_pages;
2308         else
2309                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
2310
2311         return sprintf(buf, "%lu\n", surplus_huge_pages);
2312 }
2313 HSTATE_ATTR_RO(surplus_hugepages);
2314
2315 static struct attribute *hstate_attrs[] = {
2316         &nr_hugepages_attr.attr,
2317         &nr_overcommit_hugepages_attr.attr,
2318         &free_hugepages_attr.attr,
2319         &resv_hugepages_attr.attr,
2320         &surplus_hugepages_attr.attr,
2321 #ifdef CONFIG_NUMA
2322         &nr_hugepages_mempolicy_attr.attr,
2323 #endif
2324         NULL,
2325 };
2326
2327 static struct attribute_group hstate_attr_group = {
2328         .attrs = hstate_attrs,
2329 };
2330
2331 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2332                                     struct kobject **hstate_kobjs,
2333                                     struct attribute_group *hstate_attr_group)
2334 {
2335         int retval;
2336         int hi = hstate_index(h);
2337
2338         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2339         if (!hstate_kobjs[hi])
2340                 return -ENOMEM;
2341
2342         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2343         if (retval)
2344                 kobject_put(hstate_kobjs[hi]);
2345
2346         return retval;
2347 }
2348
2349 static void __init hugetlb_sysfs_init(void)
2350 {
2351         struct hstate *h;
2352         int err;
2353
2354         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2355         if (!hugepages_kobj)
2356                 return;
2357
2358         for_each_hstate(h) {
2359                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2360                                          hstate_kobjs, &hstate_attr_group);
2361                 if (err)
2362                         pr_err("Hugetlb: Unable to add hstate %s", h->name);
2363         }
2364 }
2365
2366 #ifdef CONFIG_NUMA
2367
2368 /*
2369  * node_hstate/s - associate per node hstate attributes, via their kobjects,
2370  * with node devices in node_devices[] using a parallel array.  The array
2371  * index of a node device or _hstate == node id.
2372  * This is here to avoid any static dependency of the node device driver, in
2373  * the base kernel, on the hugetlb module.
2374  */
2375 struct node_hstate {
2376         struct kobject          *hugepages_kobj;
2377         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
2378 };
2379 struct node_hstate node_hstates[MAX_NUMNODES];
2380
2381 /*
2382  * A subset of global hstate attributes for node devices
2383  */
2384 static struct attribute *per_node_hstate_attrs[] = {
2385         &nr_hugepages_attr.attr,
2386         &free_hugepages_attr.attr,
2387         &surplus_hugepages_attr.attr,
2388         NULL,
2389 };
2390
2391 static struct attribute_group per_node_hstate_attr_group = {
2392         .attrs = per_node_hstate_attrs,
2393 };
2394
2395 /*
2396  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2397  * Returns node id via non-NULL nidp.
2398  */
2399 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2400 {
2401         int nid;
2402
2403         for (nid = 0; nid < nr_node_ids; nid++) {
2404                 struct node_hstate *nhs = &node_hstates[nid];
2405                 int i;
2406                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2407                         if (nhs->hstate_kobjs[i] == kobj) {
2408                                 if (nidp)
2409                                         *nidp = nid;
2410                                 return &hstates[i];
2411                         }
2412         }
2413
2414         BUG();
2415         return NULL;
2416 }
2417
2418 /*
2419  * Unregister hstate attributes from a single node device.
2420  * No-op if no hstate attributes attached.
2421  */
2422 static void hugetlb_unregister_node(struct node *node)
2423 {
2424         struct hstate *h;
2425         struct node_hstate *nhs = &node_hstates[node->dev.id];
2426
2427         if (!nhs->hugepages_kobj)
2428                 return;         /* no hstate attributes */
2429
2430         for_each_hstate(h) {
2431                 int idx = hstate_index(h);
2432                 if (nhs->hstate_kobjs[idx]) {
2433                         kobject_put(nhs->hstate_kobjs[idx]);
2434                         nhs->hstate_kobjs[idx] = NULL;
2435                 }
2436         }
2437
2438         kobject_put(nhs->hugepages_kobj);
2439         nhs->hugepages_kobj = NULL;
2440 }
2441
2442 /*
2443  * hugetlb module exit:  unregister hstate attributes from node devices
2444  * that have them.
2445  */
2446 static void hugetlb_unregister_all_nodes(void)
2447 {
2448         int nid;
2449
2450         /*
2451          * disable node device registrations.
2452          */
2453         register_hugetlbfs_with_node(NULL, NULL);
2454
2455         /*
2456          * remove hstate attributes from any nodes that have them.
2457          */
2458         for (nid = 0; nid < nr_node_ids; nid++)
2459                 hugetlb_unregister_node(node_devices[nid]);
2460 }
2461
2462 /*
2463  * Register hstate attributes for a single node device.
2464  * No-op if attributes already registered.
2465  */
2466 static void hugetlb_register_node(struct node *node)
2467 {
2468         struct hstate *h;
2469         struct node_hstate *nhs = &node_hstates[node->dev.id];
2470         int err;
2471
2472         if (nhs->hugepages_kobj)
2473                 return;         /* already allocated */
2474
2475         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2476                                                         &node->dev.kobj);
2477         if (!nhs->hugepages_kobj)
2478                 return;
2479
2480         for_each_hstate(h) {
2481                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2482                                                 nhs->hstate_kobjs,
2483                                                 &per_node_hstate_attr_group);
2484                 if (err) {
2485                         pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2486                                 h->name, node->dev.id);
2487                         hugetlb_unregister_node(node);
2488                         break;
2489                 }
2490         }
2491 }
2492
2493 /*
2494  * hugetlb init time:  register hstate attributes for all registered node
2495  * devices of nodes that have memory.  All on-line nodes should have
2496  * registered their associated device by this time.
2497  */
2498 static void __init hugetlb_register_all_nodes(void)
2499 {
2500         int nid;
2501
2502         for_each_node_state(nid, N_MEMORY) {
2503                 struct node *node = node_devices[nid];
2504                 if (node->dev.id == nid)
2505                         hugetlb_register_node(node);
2506         }
2507
2508         /*
2509          * Let the node device driver know we're here so it can
2510          * [un]register hstate attributes on node hotplug.
2511          */
2512         register_hugetlbfs_with_node(hugetlb_register_node,
2513                                      hugetlb_unregister_node);
2514 }
2515 #else   /* !CONFIG_NUMA */
2516
2517 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2518 {
2519         BUG();
2520         if (nidp)
2521                 *nidp = -1;
2522         return NULL;
2523 }
2524
2525 static void hugetlb_unregister_all_nodes(void) { }
2526
2527 static void hugetlb_register_all_nodes(void) { }
2528
2529 #endif
2530
2531 static void __exit hugetlb_exit(void)
2532 {
2533         struct hstate *h;
2534
2535         hugetlb_unregister_all_nodes();
2536
2537         for_each_hstate(h) {
2538                 kobject_put(hstate_kobjs[hstate_index(h)]);
2539         }
2540
2541         kobject_put(hugepages_kobj);
2542         kfree(hugetlb_fault_mutex_table);
2543 }
2544 module_exit(hugetlb_exit);
2545
2546 static int __init hugetlb_init(void)
2547 {
2548         int i;
2549
2550         if (!hugepages_supported())
2551                 return 0;
2552
2553         if (!size_to_hstate(default_hstate_size)) {
2554                 default_hstate_size = HPAGE_SIZE;
2555                 if (!size_to_hstate(default_hstate_size))
2556                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2557         }
2558         default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2559         if (default_hstate_max_huge_pages)
2560                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2561
2562         hugetlb_init_hstates();
2563         gather_bootmem_prealloc();
2564         report_hugepages();
2565
2566         hugetlb_sysfs_init();
2567         hugetlb_register_all_nodes();
2568         hugetlb_cgroup_file_init();
2569
2570 #ifdef CONFIG_SMP
2571         num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2572 #else
2573         num_fault_mutexes = 1;
2574 #endif
2575         hugetlb_fault_mutex_table =
2576                 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2577         BUG_ON(!hugetlb_fault_mutex_table);
2578
2579         for (i = 0; i < num_fault_mutexes; i++)
2580                 mutex_init(&hugetlb_fault_mutex_table[i]);
2581         return 0;
2582 }
2583 module_init(hugetlb_init);
2584
2585 /* Should be called on processing a hugepagesz=... option */
2586 void __init hugetlb_add_hstate(unsigned order)
2587 {
2588         struct hstate *h;
2589         unsigned long i;
2590
2591         if (size_to_hstate(PAGE_SIZE << order)) {
2592                 pr_warning("hugepagesz= specified twice, ignoring\n");
2593                 return;
2594         }
2595         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2596         BUG_ON(order == 0);
2597         h = &hstates[hugetlb_max_hstate++];
2598         h->order = order;
2599         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2600         h->nr_huge_pages = 0;
2601         h->free_huge_pages = 0;
2602         for (i = 0; i < MAX_NUMNODES; ++i)
2603                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2604         INIT_LIST_HEAD(&h->hugepage_activelist);
2605         h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
2606         h->next_nid_to_free = first_node(node_states[N_MEMORY]);
2607         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2608                                         huge_page_size(h)/1024);
2609
2610         parsed_hstate = h;
2611 }
2612
2613 static int __init hugetlb_nrpages_setup(char *s)
2614 {
2615         unsigned long *mhp;
2616         static unsigned long *last_mhp;
2617
2618         /*
2619          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2620          * so this hugepages= parameter goes to the "default hstate".
2621          */
2622         if (!hugetlb_max_hstate)
2623                 mhp = &default_hstate_max_huge_pages;
2624         else
2625                 mhp = &parsed_hstate->max_huge_pages;
2626
2627         if (mhp == last_mhp) {
2628                 pr_warning("hugepages= specified twice without "
2629                            "interleaving hugepagesz=, ignoring\n");
2630                 return 1;
2631         }
2632
2633         if (sscanf(s, "%lu", mhp) <= 0)
2634                 *mhp = 0;
2635
2636         /*
2637          * Global state is always initialized later in hugetlb_init.
2638          * But we need to allocate >= MAX_ORDER hstates here early to still
2639          * use the bootmem allocator.
2640          */
2641         if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2642                 hugetlb_hstate_alloc_pages(parsed_hstate);
2643
2644         last_mhp = mhp;
2645
2646         return 1;
2647 }
2648 __setup("hugepages=", hugetlb_nrpages_setup);
2649
2650 static int __init hugetlb_default_setup(char *s)
2651 {
2652         default_hstate_size = memparse(s, &s);
2653         return 1;
2654 }
2655 __setup("default_hugepagesz=", hugetlb_default_setup);
2656
2657 static unsigned int cpuset_mems_nr(unsigned int *array)
2658 {
2659         int node;
2660         unsigned int nr = 0;
2661
2662         for_each_node_mask(node, cpuset_current_mems_allowed)
2663                 nr += array[node];
2664
2665         return nr;
2666 }
2667
2668 #ifdef CONFIG_SYSCTL
2669 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2670                          struct ctl_table *table, int write,
2671                          void __user *buffer, size_t *length, loff_t *ppos)
2672 {
2673         struct hstate *h = &default_hstate;
2674         unsigned long tmp = h->max_huge_pages;
2675         int ret;
2676
2677         if (!hugepages_supported())
2678                 return -ENOTSUPP;
2679
2680         table->data = &tmp;
2681         table->maxlen = sizeof(unsigned long);
2682         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2683         if (ret)
2684                 goto out;
2685
2686         if (write)
2687                 ret = __nr_hugepages_store_common(obey_mempolicy, h,
2688                                                   NUMA_NO_NODE, tmp, *length);
2689 out:
2690         return ret;
2691 }
2692
2693 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2694                           void __user *buffer, size_t *length, loff_t *ppos)
2695 {
2696
2697         return hugetlb_sysctl_handler_common(false, table, write,
2698                                                         buffer, length, ppos);
2699 }
2700
2701 #ifdef CONFIG_NUMA
2702 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2703                           void __user *buffer, size_t *length, loff_t *ppos)
2704 {
2705         return hugetlb_sysctl_handler_common(true, table, write,
2706                                                         buffer, length, ppos);
2707 }
2708 #endif /* CONFIG_NUMA */
2709
2710 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2711                         void __user *buffer,
2712                         size_t *length, loff_t *ppos)
2713 {
2714         struct hstate *h = &default_hstate;
2715         unsigned long tmp;
2716         int ret;
2717
2718         if (!hugepages_supported())
2719                 return -ENOTSUPP;
2720
2721         tmp = h->nr_overcommit_huge_pages;
2722
2723         if (write && hstate_is_gigantic(h))
2724                 return -EINVAL;
2725
2726         table->data = &tmp;
2727         table->maxlen = sizeof(unsigned long);
2728         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2729         if (ret)
2730                 goto out;
2731
2732         if (write) {
2733                 spin_lock(&hugetlb_lock);
2734                 h->nr_overcommit_huge_pages = tmp;
2735                 spin_unlock(&hugetlb_lock);
2736         }
2737 out:
2738         return ret;
2739 }
2740
2741 #endif /* CONFIG_SYSCTL */
2742
2743 void hugetlb_report_meminfo(struct seq_file *m)
2744 {
2745         struct hstate *h = &default_hstate;
2746         if (!hugepages_supported())
2747                 return;
2748         seq_printf(m,
2749                         "HugePages_Total:   %5lu\n"
2750                         "HugePages_Free:    %5lu\n"
2751                         "HugePages_Rsvd:    %5lu\n"
2752                         "HugePages_Surp:    %5lu\n"
2753                         "Hugepagesize:   %8lu kB\n",
2754                         h->nr_huge_pages,
2755                         h->free_huge_pages,
2756                         h->resv_huge_pages,
2757                         h->surplus_huge_pages,
2758                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2759 }
2760
2761 int hugetlb_report_node_meminfo(int nid, char *buf)
2762 {
2763         struct hstate *h = &default_hstate;
2764         if (!hugepages_supported())
2765                 return 0;
2766         return sprintf(buf,
2767                 "Node %d HugePages_Total: %5u\n"
2768                 "Node %d HugePages_Free:  %5u\n"
2769                 "Node %d HugePages_Surp:  %5u\n",
2770                 nid, h->nr_huge_pages_node[nid],
2771                 nid, h->free_huge_pages_node[nid],
2772                 nid, h->surplus_huge_pages_node[nid]);
2773 }
2774
2775 void hugetlb_show_meminfo(void)
2776 {
2777         struct hstate *h;
2778         int nid;
2779
2780         if (!hugepages_supported())
2781                 return;
2782
2783         for_each_node_state(nid, N_MEMORY)
2784                 for_each_hstate(h)
2785                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2786                                 nid,
2787                                 h->nr_huge_pages_node[nid],
2788                                 h->free_huge_pages_node[nid],
2789                                 h->surplus_huge_pages_node[nid],
2790                                 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2791 }
2792
2793 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2794 unsigned long hugetlb_total_pages(void)
2795 {
2796         struct hstate *h;
2797         unsigned long nr_total_pages = 0;
2798
2799         for_each_hstate(h)
2800                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2801         return nr_total_pages;
2802 }
2803
2804 static int hugetlb_acct_memory(struct hstate *h, long delta)
2805 {
2806         int ret = -ENOMEM;
2807
2808         spin_lock(&hugetlb_lock);
2809         /*
2810          * When cpuset is configured, it breaks the strict hugetlb page
2811          * reservation as the accounting is done on a global variable. Such
2812          * reservation is completely rubbish in the presence of cpuset because
2813          * the reservation is not checked against page availability for the
2814          * current cpuset. Application can still potentially OOM'ed by kernel
2815          * with lack of free htlb page in cpuset that the task is in.
2816          * Attempt to enforce strict accounting with cpuset is almost
2817          * impossible (or too ugly) because cpuset is too fluid that
2818          * task or memory node can be dynamically moved between cpusets.
2819          *
2820          * The change of semantics for shared hugetlb mapping with cpuset is
2821          * undesirable. However, in order to preserve some of the semantics,
2822          * we fall back to check against current free page availability as
2823          * a best attempt and hopefully to minimize the impact of changing
2824          * semantics that cpuset has.
2825          */
2826         if (delta > 0) {
2827                 if (gather_surplus_pages(h, delta) < 0)
2828                         goto out;
2829
2830                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2831                         return_unused_surplus_pages(h, delta);
2832                         goto out;
2833                 }
2834         }
2835
2836         ret = 0;
2837         if (delta < 0)
2838                 return_unused_surplus_pages(h, (unsigned long) -delta);
2839
2840 out:
2841         spin_unlock(&hugetlb_lock);
2842         return ret;
2843 }
2844
2845 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2846 {
2847         struct resv_map *resv = vma_resv_map(vma);
2848
2849         /*
2850          * This new VMA should share its siblings reservation map if present.
2851          * The VMA will only ever have a valid reservation map pointer where
2852          * it is being copied for another still existing VMA.  As that VMA
2853          * has a reference to the reservation map it cannot disappear until
2854          * after this open call completes.  It is therefore safe to take a
2855          * new reference here without additional locking.
2856          */
2857         if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2858                 kref_get(&resv->refs);
2859 }
2860
2861 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2862 {
2863         struct hstate *h = hstate_vma(vma);
2864         struct resv_map *resv = vma_resv_map(vma);
2865         struct hugepage_subpool *spool = subpool_vma(vma);
2866         unsigned long reserve, start, end;
2867         long gbl_reserve;
2868
2869         if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2870                 return;
2871
2872         start = vma_hugecache_offset(h, vma, vma->vm_start);
2873         end = vma_hugecache_offset(h, vma, vma->vm_end);
2874
2875         reserve = (end - start) - region_count(resv, start, end);
2876
2877         kref_put(&resv->refs, resv_map_release);
2878
2879         if (reserve) {
2880                 /*
2881                  * Decrement reserve counts.  The global reserve count may be
2882                  * adjusted if the subpool has a minimum size.
2883                  */
2884                 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
2885                 hugetlb_acct_memory(h, -gbl_reserve);
2886         }
2887 }
2888
2889 /*
2890  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2891  * handle_mm_fault() to try to instantiate regular-sized pages in the
2892  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2893  * this far.
2894  */
2895 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2896 {
2897         BUG();
2898         return 0;
2899 }
2900
2901 const struct vm_operations_struct hugetlb_vm_ops = {
2902         .fault = hugetlb_vm_op_fault,
2903         .open = hugetlb_vm_op_open,
2904         .close = hugetlb_vm_op_close,
2905 };
2906
2907 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2908                                 int writable)
2909 {
2910         pte_t entry;
2911
2912         if (writable) {
2913                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2914                                          vma->vm_page_prot)));
2915         } else {
2916                 entry = huge_pte_wrprotect(mk_huge_pte(page,
2917                                            vma->vm_page_prot));
2918         }
2919         entry = pte_mkyoung(entry);
2920         entry = pte_mkhuge(entry);
2921         entry = arch_make_huge_pte(entry, vma, page, writable);
2922
2923         return entry;
2924 }
2925
2926 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2927                                    unsigned long address, pte_t *ptep)
2928 {
2929         pte_t entry;
2930
2931         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
2932         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2933                 update_mmu_cache(vma, address, ptep);
2934 }
2935
2936 static int is_hugetlb_entry_migration(pte_t pte)
2937 {
2938         swp_entry_t swp;
2939
2940         if (huge_pte_none(pte) || pte_present(pte))
2941                 return 0;
2942         swp = pte_to_swp_entry(pte);
2943         if (non_swap_entry(swp) && is_migration_entry(swp))
2944                 return 1;
2945         else
2946                 return 0;
2947 }
2948
2949 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2950 {
2951         swp_entry_t swp;
2952
2953         if (huge_pte_none(pte) || pte_present(pte))
2954                 return 0;
2955         swp = pte_to_swp_entry(pte);
2956         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2957                 return 1;
2958         else
2959                 return 0;
2960 }
2961
2962 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2963                             struct vm_area_struct *vma)
2964 {
2965         pte_t *src_pte, *dst_pte, entry;
2966         struct page *ptepage;
2967         unsigned long addr;
2968         int cow;
2969         struct hstate *h = hstate_vma(vma);
2970         unsigned long sz = huge_page_size(h);
2971         unsigned long mmun_start;       /* For mmu_notifiers */
2972         unsigned long mmun_end;         /* For mmu_notifiers */
2973         int ret = 0;
2974
2975         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2976
2977         mmun_start = vma->vm_start;
2978         mmun_end = vma->vm_end;
2979         if (cow)
2980                 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
2981
2982         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2983                 spinlock_t *src_ptl, *dst_ptl;
2984                 src_pte = huge_pte_offset(src, addr);
2985                 if (!src_pte)
2986                         continue;
2987                 dst_pte = huge_pte_alloc(dst, addr, sz);
2988                 if (!dst_pte) {
2989                         ret = -ENOMEM;
2990                         break;
2991                 }
2992
2993                 /* If the pagetables are shared don't copy or take references */
2994                 if (dst_pte == src_pte)
2995                         continue;
2996
2997                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
2998                 src_ptl = huge_pte_lockptr(h, src, src_pte);
2999                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3000                 entry = huge_ptep_get(src_pte);
3001                 if (huge_pte_none(entry)) { /* skip none entry */
3002                         ;
3003                 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
3004                                     is_hugetlb_entry_hwpoisoned(entry))) {
3005                         swp_entry_t swp_entry = pte_to_swp_entry(entry);
3006
3007                         if (is_write_migration_entry(swp_entry) && cow) {
3008                                 /*
3009                                  * COW mappings require pages in both
3010                                  * parent and child to be set to read.
3011                                  */
3012                                 make_migration_entry_read(&swp_entry);
3013                                 entry = swp_entry_to_pte(swp_entry);
3014                                 set_huge_pte_at(src, addr, src_pte, entry);
3015                         }
3016                         set_huge_pte_at(dst, addr, dst_pte, entry);
3017                 } else {
3018                         if (cow) {
3019                                 huge_ptep_set_wrprotect(src, addr, src_pte);
3020                                 mmu_notifier_invalidate_range(src, mmun_start,
3021                                                                    mmun_end);
3022                         }
3023                         entry = huge_ptep_get(src_pte);
3024                         ptepage = pte_page(entry);
3025                         get_page(ptepage);
3026                         page_dup_rmap(ptepage);
3027                         set_huge_pte_at(dst, addr, dst_pte, entry);
3028                 }
3029                 spin_unlock(src_ptl);
3030                 spin_unlock(dst_ptl);
3031         }
3032
3033         if (cow)
3034                 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
3035
3036         return ret;
3037 }
3038
3039 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
3040                             unsigned long start, unsigned long end,
3041                             struct page *ref_page)
3042 {
3043         int force_flush = 0;
3044         struct mm_struct *mm = vma->vm_mm;
3045         unsigned long address;
3046         pte_t *ptep;
3047         pte_t pte;
3048         spinlock_t *ptl;
3049         struct page *page;
3050         struct hstate *h = hstate_vma(vma);
3051         unsigned long sz = huge_page_size(h);
3052         const unsigned long mmun_start = start; /* For mmu_notifiers */
3053         const unsigned long mmun_end   = end;   /* For mmu_notifiers */
3054
3055         WARN_ON(!is_vm_hugetlb_page(vma));
3056         BUG_ON(start & ~huge_page_mask(h));
3057         BUG_ON(end & ~huge_page_mask(h));
3058
3059         tlb_start_vma(tlb, vma);
3060         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3061         address = start;
3062 again:
3063         for (; address < end; address += sz) {
3064                 ptep = huge_pte_offset(mm, address);
3065                 if (!ptep)
3066                         continue;
3067
3068                 ptl = huge_pte_lock(h, mm, ptep);
3069                 if (huge_pmd_unshare(mm, &address, ptep))
3070                         goto unlock;
3071
3072                 pte = huge_ptep_get(ptep);
3073                 if (huge_pte_none(pte))
3074                         goto unlock;
3075
3076                 /*
3077                  * Migrating hugepage or HWPoisoned hugepage is already
3078                  * unmapped and its refcount is dropped, so just clear pte here.
3079                  */
3080                 if (unlikely(!pte_present(pte))) {
3081                         huge_pte_clear(mm, address, ptep);
3082                         goto unlock;
3083                 }
3084
3085                 page = pte_page(pte);
3086                 /*
3087                  * If a reference page is supplied, it is because a specific
3088                  * page is being unmapped, not a range. Ensure the page we
3089                  * are about to unmap is the actual page of interest.
3090                  */
3091                 if (ref_page) {
3092                         if (page != ref_page)
3093                                 goto unlock;
3094
3095                         /*
3096                          * Mark the VMA as having unmapped its page so that
3097                          * future faults in this VMA will fail rather than
3098                          * looking like data was lost
3099                          */
3100                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
3101                 }
3102
3103                 pte = huge_ptep_get_and_clear(mm, address, ptep);
3104                 tlb_remove_tlb_entry(tlb, ptep, address);
3105                 if (huge_pte_dirty(pte))
3106                         set_page_dirty(page);
3107
3108                 page_remove_rmap(page);
3109                 force_flush = !__tlb_remove_page(tlb, page);
3110                 if (force_flush) {
3111                         address += sz;
3112                         spin_unlock(ptl);
3113                         break;
3114                 }
3115                 /* Bail out after unmapping reference page if supplied */
3116                 if (ref_page) {
3117                         spin_unlock(ptl);
3118                         break;
3119                 }
3120 unlock:
3121                 spin_unlock(ptl);
3122         }
3123         /*
3124          * mmu_gather ran out of room to batch pages, we break out of
3125          * the PTE lock to avoid doing the potential expensive TLB invalidate
3126          * and page-free while holding it.
3127          */
3128         if (force_flush) {
3129                 force_flush = 0;
3130                 tlb_flush_mmu(tlb);
3131                 if (address < end && !ref_page)
3132                         goto again;
3133         }
3134         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3135         tlb_end_vma(tlb, vma);
3136 }
3137
3138 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
3139                           struct vm_area_struct *vma, unsigned long start,
3140                           unsigned long end, struct page *ref_page)
3141 {
3142         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
3143
3144         /*
3145          * Clear this flag so that x86's huge_pmd_share page_table_shareable
3146          * test will fail on a vma being torn down, and not grab a page table
3147          * on its way out.  We're lucky that the flag has such an appropriate
3148          * name, and can in fact be safely cleared here. We could clear it
3149          * before the __unmap_hugepage_range above, but all that's necessary
3150          * is to clear it before releasing the i_mmap_rwsem. This works
3151          * because in the context this is called, the VMA is about to be
3152          * destroyed and the i_mmap_rwsem is held.
3153          */
3154         vma->vm_flags &= ~VM_MAYSHARE;
3155 }
3156
3157 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
3158                           unsigned long end, struct page *ref_page)
3159 {
3160         struct mm_struct *mm;
3161         struct mmu_gather tlb;
3162
3163         mm = vma->vm_mm;
3164
3165         tlb_gather_mmu(&tlb, mm, start, end);
3166         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3167         tlb_finish_mmu(&tlb, start, end);
3168 }
3169
3170 /*
3171  * This is called when the original mapper is failing to COW a MAP_PRIVATE
3172  * mappping it owns the reserve page for. The intention is to unmap the page
3173  * from other VMAs and let the children be SIGKILLed if they are faulting the
3174  * same region.
3175  */
3176 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
3177                               struct page *page, unsigned long address)
3178 {
3179         struct hstate *h = hstate_vma(vma);
3180         struct vm_area_struct *iter_vma;
3181         struct address_space *mapping;
3182         pgoff_t pgoff;
3183
3184         /*
3185          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
3186          * from page cache lookup which is in HPAGE_SIZE units.
3187          */
3188         address = address & huge_page_mask(h);
3189         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
3190                         vma->vm_pgoff;
3191         mapping = file_inode(vma->vm_file)->i_mapping;
3192
3193         /*
3194          * Take the mapping lock for the duration of the table walk. As
3195          * this mapping should be shared between all the VMAs,
3196          * __unmap_hugepage_range() is called as the lock is already held
3197          */
3198         i_mmap_lock_write(mapping);
3199         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
3200                 /* Do not unmap the current VMA */
3201                 if (iter_vma == vma)
3202                         continue;
3203
3204                 /*
3205                  * Shared VMAs have their own reserves and do not affect
3206                  * MAP_PRIVATE accounting but it is possible that a shared
3207                  * VMA is using the same page so check and skip such VMAs.
3208                  */
3209                 if (iter_vma->vm_flags & VM_MAYSHARE)
3210                         continue;
3211
3212                 /*
3213                  * Unmap the page from other VMAs without their own reserves.
3214                  * They get marked to be SIGKILLed if they fault in these
3215                  * areas. This is because a future no-page fault on this VMA
3216                  * could insert a zeroed page instead of the data existing
3217                  * from the time of fork. This would look like data corruption
3218                  */
3219                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
3220                         unmap_hugepage_range(iter_vma, address,
3221                                              address + huge_page_size(h), page);
3222         }
3223         i_mmap_unlock_write(mapping);
3224 }
3225
3226 /*
3227  * Hugetlb_cow() should be called with page lock of the original hugepage held.
3228  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
3229  * cannot race with other handlers or page migration.
3230  * Keep the pte_same checks anyway to make transition from the mutex easier.
3231  */
3232 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
3233                         unsigned long address, pte_t *ptep, pte_t pte,
3234                         struct page *pagecache_page, spinlock_t *ptl)
3235 {
3236         struct hstate *h = hstate_vma(vma);
3237         struct page *old_page, *new_page;
3238         int ret = 0, outside_reserve = 0;
3239         unsigned long mmun_start;       /* For mmu_notifiers */
3240         unsigned long mmun_end;         /* For mmu_notifiers */
3241
3242         old_page = pte_page(pte);
3243
3244 retry_avoidcopy:
3245         /* If no-one else is actually using this page, avoid the copy
3246          * and just make the page writable */
3247         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
3248                 page_move_anon_rmap(old_page, vma, address);
3249                 set_huge_ptep_writable(vma, address, ptep);
3250                 return 0;
3251         }
3252
3253         /*
3254          * If the process that created a MAP_PRIVATE mapping is about to
3255          * perform a COW due to a shared page count, attempt to satisfy
3256          * the allocation without using the existing reserves. The pagecache
3257          * page is used to determine if the reserve at this address was
3258          * consumed or not. If reserves were used, a partial faulted mapping
3259          * at the time of fork() could consume its reserves on COW instead
3260          * of the full address range.
3261          */
3262         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
3263                         old_page != pagecache_page)
3264                 outside_reserve = 1;
3265
3266         page_cache_get(old_page);
3267
3268         /*
3269          * Drop page table lock as buddy allocator may be called. It will
3270          * be acquired again before returning to the caller, as expected.
3271          */
3272         spin_unlock(ptl);
3273         new_page = alloc_huge_page(vma, address, outside_reserve);
3274
3275         if (IS_ERR(new_page)) {
3276                 /*
3277                  * If a process owning a MAP_PRIVATE mapping fails to COW,
3278                  * it is due to references held by a child and an insufficient
3279                  * huge page pool. To guarantee the original mappers
3280                  * reliability, unmap the page from child processes. The child
3281                  * may get SIGKILLed if it later faults.
3282                  */
3283                 if (outside_reserve) {
3284                         page_cache_release(old_page);
3285                         BUG_ON(huge_pte_none(pte));
3286                         unmap_ref_private(mm, vma, old_page, address);
3287                         BUG_ON(huge_pte_none(pte));
3288                         spin_lock(ptl);
3289                         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3290                         if (likely(ptep &&
3291                                    pte_same(huge_ptep_get(ptep), pte)))
3292                                 goto retry_avoidcopy;
3293                         /*
3294                          * race occurs while re-acquiring page table
3295                          * lock, and our job is done.
3296                          */
3297                         return 0;
3298                 }
3299
3300                 ret = (PTR_ERR(new_page) == -ENOMEM) ?
3301                         VM_FAULT_OOM : VM_FAULT_SIGBUS;
3302                 goto out_release_old;
3303         }
3304
3305         /*
3306          * When the original hugepage is shared one, it does not have
3307          * anon_vma prepared.
3308          */
3309         if (unlikely(anon_vma_prepare(vma))) {
3310                 ret = VM_FAULT_OOM;
3311                 goto out_release_all;
3312         }
3313
3314         copy_user_huge_page(new_page, old_page, address, vma,
3315                             pages_per_huge_page(h));
3316         __SetPageUptodate(new_page);
3317         set_page_huge_active(new_page);
3318
3319         mmun_start = address & huge_page_mask(h);
3320         mmun_end = mmun_start + huge_page_size(h);
3321         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3322
3323         /*
3324          * Retake the page table lock to check for racing updates
3325          * before the page tables are altered
3326          */
3327         spin_lock(ptl);
3328         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3329         if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3330                 ClearPagePrivate(new_page);
3331
3332                 /* Break COW */
3333                 huge_ptep_clear_flush(vma, address, ptep);
3334                 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
3335                 set_huge_pte_at(mm, address, ptep,
3336                                 make_huge_pte(vma, new_page, 1));
3337                 page_remove_rmap(old_page);
3338                 hugepage_add_new_anon_rmap(new_page, vma, address);
3339                 /* Make the old page be freed below */
3340                 new_page = old_page;
3341         }
3342         spin_unlock(ptl);
3343         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3344 out_release_all:
3345         page_cache_release(new_page);
3346 out_release_old:
3347         page_cache_release(old_page);
3348
3349         spin_lock(ptl); /* Caller expects lock to be held */
3350         return ret;
3351 }
3352
3353 /* Return the pagecache page at a given address within a VMA */
3354 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3355                         struct vm_area_struct *vma, unsigned long address)
3356 {
3357         struct address_space *mapping;
3358         pgoff_t idx;
3359
3360         mapping = vma->vm_file->f_mapping;
3361         idx = vma_hugecache_offset(h, vma, address);
3362
3363         return find_lock_page(mapping, idx);
3364 }
3365
3366 /*
3367  * Return whether there is a pagecache page to back given address within VMA.
3368  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3369  */
3370 static bool hugetlbfs_pagecache_present(struct hstate *h,
3371                         struct vm_area_struct *vma, unsigned long address)
3372 {
3373         struct address_space *mapping;
3374         pgoff_t idx;
3375         struct page *page;
3376
3377         mapping = vma->vm_file->f_mapping;
3378         idx = vma_hugecache_offset(h, vma, address);
3379
3380         page = find_get_page(mapping, idx);
3381         if (page)
3382                 put_page(page);
3383         return page != NULL;
3384 }
3385
3386 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
3387                            pgoff_t idx)
3388 {
3389         struct inode *inode = mapping->host;
3390         struct hstate *h = hstate_inode(inode);
3391         int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3392
3393         if (err)
3394                 return err;
3395         ClearPagePrivate(page);
3396
3397         spin_lock(&inode->i_lock);
3398         inode->i_blocks += blocks_per_huge_page(h);
3399         spin_unlock(&inode->i_lock);
3400         return 0;
3401 }
3402
3403 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3404                            struct address_space *mapping, pgoff_t idx,
3405                            unsigned long address, pte_t *ptep, unsigned int flags)
3406 {
3407         struct hstate *h = hstate_vma(vma);
3408         int ret = VM_FAULT_SIGBUS;
3409         int anon_rmap = 0;
3410         unsigned long size;
3411         struct page *page;
3412         pte_t new_pte;
3413         spinlock_t *ptl;
3414
3415         /*
3416          * Currently, we are forced to kill the process in the event the
3417          * original mapper has unmapped pages from the child due to a failed
3418          * COW. Warn that such a situation has occurred as it may not be obvious
3419          */
3420         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3421                 pr_warning("PID %d killed due to inadequate hugepage pool\n",
3422                            current->pid);
3423                 return ret;
3424         }
3425
3426         /*
3427          * Use page lock to guard against racing truncation
3428          * before we get page_table_lock.
3429          */
3430 retry:
3431         page = find_lock_page(mapping, idx);
3432         if (!page) {
3433                 size = i_size_read(mapping->host) >> huge_page_shift(h);
3434                 if (idx >= size)
3435                         goto out;
3436                 page = alloc_huge_page(vma, address, 0);
3437                 if (IS_ERR(page)) {
3438                         ret = PTR_ERR(page);
3439                         if (ret == -ENOMEM)
3440                                 ret = VM_FAULT_OOM;
3441                         else
3442                                 ret = VM_FAULT_SIGBUS;
3443                         goto out;
3444                 }
3445                 clear_huge_page(page, address, pages_per_huge_page(h));
3446                 __SetPageUptodate(page);
3447                 set_page_huge_active(page);
3448
3449                 if (vma->vm_flags & VM_MAYSHARE) {
3450                         int err = huge_add_to_page_cache(page, mapping, idx);
3451                         if (err) {
3452                                 put_page(page);
3453                                 if (err == -EEXIST)
3454                                         goto retry;
3455                                 goto out;
3456                         }
3457                 } else {
3458                         lock_page(page);
3459                         if (unlikely(anon_vma_prepare(vma))) {
3460                                 ret = VM_FAULT_OOM;
3461                                 goto backout_unlocked;
3462                         }
3463                         anon_rmap = 1;
3464                 }
3465         } else {
3466                 /*
3467                  * If memory error occurs between mmap() and fault, some process
3468                  * don't have hwpoisoned swap entry for errored virtual address.
3469                  * So we need to block hugepage fault by PG_hwpoison bit check.
3470                  */
3471                 if (unlikely(PageHWPoison(page))) {
3472                         ret = VM_FAULT_HWPOISON |
3473                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3474                         goto backout_unlocked;
3475                 }
3476         }
3477
3478         /*
3479          * If we are going to COW a private mapping later, we examine the
3480          * pending reservations for this page now. This will ensure that
3481          * any allocations necessary to record that reservation occur outside
3482          * the spinlock.
3483          */
3484         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3485                 if (vma_needs_reservation(h, vma, address) < 0) {
3486                         ret = VM_FAULT_OOM;
3487                         goto backout_unlocked;
3488                 }
3489                 /* Just decrements count, does not deallocate */
3490                 vma_end_reservation(h, vma, address);
3491         }
3492
3493         ptl = huge_pte_lockptr(h, mm, ptep);
3494         spin_lock(ptl);
3495         size = i_size_read(mapping->host) >> huge_page_shift(h);
3496         if (idx >= size)
3497                 goto backout;
3498
3499         ret = 0;
3500         if (!huge_pte_none(huge_ptep_get(ptep)))
3501                 goto backout;
3502
3503         if (anon_rmap) {
3504                 ClearPagePrivate(page);
3505                 hugepage_add_new_anon_rmap(page, vma, address);
3506         } else
3507                 page_dup_rmap(page);
3508         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3509                                 && (vma->vm_flags & VM_SHARED)));
3510         set_huge_pte_at(mm, address, ptep, new_pte);
3511
3512         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3513                 /* Optimization, do the COW without a second fault */
3514                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
3515         }
3516
3517         spin_unlock(ptl);
3518         unlock_page(page);
3519 out:
3520         return ret;
3521
3522 backout:
3523         spin_unlock(ptl);
3524 backout_unlocked:
3525         unlock_page(page);
3526         put_page(page);
3527         goto out;
3528 }
3529
3530 #ifdef CONFIG_SMP
3531 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3532                             struct vm_area_struct *vma,
3533                             struct address_space *mapping,
3534                             pgoff_t idx, unsigned long address)
3535 {
3536         unsigned long key[2];
3537         u32 hash;
3538
3539         if (vma->vm_flags & VM_SHARED) {
3540                 key[0] = (unsigned long) mapping;
3541                 key[1] = idx;
3542         } else {
3543                 key[0] = (unsigned long) mm;
3544                 key[1] = address >> huge_page_shift(h);
3545         }
3546
3547         hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3548
3549         return hash & (num_fault_mutexes - 1);
3550 }
3551 #else
3552 /*
3553  * For uniprocesor systems we always use a single mutex, so just
3554  * return 0 and avoid the hashing overhead.
3555  */
3556 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3557                             struct vm_area_struct *vma,
3558                             struct address_space *mapping,
3559                             pgoff_t idx, unsigned long address)
3560 {
3561         return 0;
3562 }
3563 #endif
3564
3565 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3566                         unsigned long address, unsigned int flags)
3567 {
3568         pte_t *ptep, entry;
3569         spinlock_t *ptl;
3570         int ret;
3571         u32 hash;
3572         pgoff_t idx;
3573         struct page *page = NULL;
3574         struct page *pagecache_page = NULL;
3575         struct hstate *h = hstate_vma(vma);
3576         struct address_space *mapping;
3577         int need_wait_lock = 0;
3578
3579         address &= huge_page_mask(h);
3580
3581         ptep = huge_pte_offset(mm, address);
3582         if (ptep) {
3583                 entry = huge_ptep_get(ptep);
3584                 if (unlikely(is_hugetlb_entry_migration(entry))) {
3585                         migration_entry_wait_huge(vma, mm, ptep);
3586                         return 0;
3587                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3588                         return VM_FAULT_HWPOISON_LARGE |
3589                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3590         }
3591
3592         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3593         if (!ptep)
3594                 return VM_FAULT_OOM;
3595
3596         mapping = vma->vm_file->f_mapping;
3597         idx = vma_hugecache_offset(h, vma, address);
3598
3599         /*
3600          * Serialize hugepage allocation and instantiation, so that we don't
3601          * get spurious allocation failures if two CPUs race to instantiate
3602          * the same page in the page cache.
3603          */
3604         hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address);
3605         mutex_lock(&hugetlb_fault_mutex_table[hash]);
3606
3607         entry = huge_ptep_get(ptep);
3608         if (huge_pte_none(entry)) {
3609                 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3610                 goto out_mutex;
3611         }
3612
3613         ret = 0;
3614
3615         /*
3616          * entry could be a migration/hwpoison entry at this point, so this
3617          * check prevents the kernel from going below assuming that we have
3618          * a active hugepage in pagecache. This goto expects the 2nd page fault,
3619          * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3620          * handle it.
3621          */
3622         if (!pte_present(entry))
3623                 goto out_mutex;
3624
3625         /*
3626          * If we are going to COW the mapping later, we examine the pending
3627          * reservations for this page now. This will ensure that any
3628          * allocations necessary to record that reservation occur outside the
3629          * spinlock. For private mappings, we also lookup the pagecache
3630          * page now as it is used to determine if a reservation has been
3631          * consumed.
3632          */
3633         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3634                 if (vma_needs_reservation(h, vma, address) < 0) {
3635                         ret = VM_FAULT_OOM;
3636                         goto out_mutex;
3637                 }
3638                 /* Just decrements count, does not deallocate */
3639                 vma_end_reservation(h, vma, address);
3640
3641                 if (!(vma->vm_flags & VM_MAYSHARE))
3642                         pagecache_page = hugetlbfs_pagecache_page(h,
3643                                                                 vma, address);
3644         }
3645
3646         ptl = huge_pte_lock(h, mm, ptep);
3647
3648         /* Check for a racing update before calling hugetlb_cow */
3649         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3650                 goto out_ptl;
3651
3652         /*
3653          * hugetlb_cow() requires page locks of pte_page(entry) and
3654          * pagecache_page, so here we need take the former one
3655          * when page != pagecache_page or !pagecache_page.
3656          */
3657         page = pte_page(entry);
3658         if (page != pagecache_page)
3659                 if (!trylock_page(page)) {
3660                         need_wait_lock = 1;
3661                         goto out_ptl;
3662                 }
3663
3664         get_page(page);
3665
3666         if (flags & FAULT_FLAG_WRITE) {
3667                 if (!huge_pte_write(entry)) {
3668                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
3669                                         pagecache_page, ptl);
3670                         goto out_put_page;
3671                 }
3672                 entry = huge_pte_mkdirty(entry);
3673         }
3674         entry = pte_mkyoung(entry);
3675         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3676                                                 flags & FAULT_FLAG_WRITE))
3677                 update_mmu_cache(vma, address, ptep);
3678 out_put_page:
3679         if (page != pagecache_page)
3680                 unlock_page(page);
3681         put_page(page);
3682 out_ptl:
3683         spin_unlock(ptl);
3684
3685         if (pagecache_page) {
3686                 unlock_page(pagecache_page);
3687                 put_page(pagecache_page);
3688         }
3689 out_mutex:
3690         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
3691         /*
3692          * Generally it's safe to hold refcount during waiting page lock. But
3693          * here we just wait to defer the next page fault to avoid busy loop and
3694          * the page is not used after unlocked before returning from the current
3695          * page fault. So we are safe from accessing freed page, even if we wait
3696          * here without taking refcount.
3697          */
3698         if (need_wait_lock)
3699                 wait_on_page_locked(page);
3700         return ret;
3701 }
3702
3703 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3704                          struct page **pages, struct vm_area_struct **vmas,
3705                          unsigned long *position, unsigned long *nr_pages,
3706                          long i, unsigned int flags)
3707 {
3708         unsigned long pfn_offset;
3709         unsigned long vaddr = *position;
3710         unsigned long remainder = *nr_pages;
3711         struct hstate *h = hstate_vma(vma);
3712
3713         while (vaddr < vma->vm_end && remainder) {
3714                 pte_t *pte;
3715                 spinlock_t *ptl = NULL;
3716                 int absent;
3717                 struct page *page;
3718
3719                 /*
3720                  * If we have a pending SIGKILL, don't keep faulting pages and
3721                  * potentially allocating memory.
3722                  */
3723                 if (unlikely(fatal_signal_pending(current))) {
3724                         remainder = 0;
3725                         break;
3726                 }
3727
3728                 /*
3729                  * Some archs (sparc64, sh*) have multiple pte_ts to
3730                  * each hugepage.  We have to make sure we get the
3731                  * first, for the page indexing below to work.
3732                  *
3733                  * Note that page table lock is not held when pte is null.
3734                  */
3735                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3736                 if (pte)
3737                         ptl = huge_pte_lock(h, mm, pte);
3738                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3739
3740                 /*
3741                  * When coredumping, it suits get_dump_page if we just return
3742                  * an error where there's an empty slot with no huge pagecache
3743                  * to back it.  This way, we avoid allocating a hugepage, and
3744                  * the sparse dumpfile avoids allocating disk blocks, but its
3745                  * huge holes still show up with zeroes where they need to be.
3746                  */
3747                 if (absent && (flags & FOLL_DUMP) &&
3748                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3749                         if (pte)
3750                                 spin_unlock(ptl);
3751                         remainder = 0;
3752                         break;
3753                 }
3754
3755                 /*
3756                  * We need call hugetlb_fault for both hugepages under migration
3757                  * (in which case hugetlb_fault waits for the migration,) and
3758                  * hwpoisoned hugepages (in which case we need to prevent the
3759                  * caller from accessing to them.) In order to do this, we use
3760                  * here is_swap_pte instead of is_hugetlb_entry_migration and
3761                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3762                  * both cases, and because we can't follow correct pages
3763                  * directly from any kind of swap entries.
3764                  */
3765                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3766                     ((flags & FOLL_WRITE) &&
3767                       !huge_pte_write(huge_ptep_get(pte)))) {
3768                         int ret;
3769
3770                         if (pte)
3771                                 spin_unlock(ptl);
3772                         ret = hugetlb_fault(mm, vma, vaddr,
3773                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3774                         if (!(ret & VM_FAULT_ERROR))
3775                                 continue;
3776
3777                         remainder = 0;
3778                         break;
3779                 }
3780
3781                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3782                 page = pte_page(huge_ptep_get(pte));
3783 same_page:
3784                 if (pages) {
3785                         pages[i] = mem_map_offset(page, pfn_offset);
3786                         get_page_foll(pages[i]);
3787                 }
3788
3789                 if (vmas)
3790                         vmas[i] = vma;
3791
3792                 vaddr += PAGE_SIZE;
3793                 ++pfn_offset;
3794                 --remainder;
3795                 ++i;
3796                 if (vaddr < vma->vm_end && remainder &&
3797                                 pfn_offset < pages_per_huge_page(h)) {
3798                         /*
3799                          * We use pfn_offset to avoid touching the pageframes
3800                          * of this compound page.
3801                          */
3802                         goto same_page;
3803                 }
3804                 spin_unlock(ptl);
3805         }
3806         *nr_pages = remainder;
3807         *position = vaddr;
3808
3809         return i ? i : -EFAULT;
3810 }
3811
3812 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3813                 unsigned long address, unsigned long end, pgprot_t newprot)
3814 {
3815         struct mm_struct *mm = vma->vm_mm;
3816         unsigned long start = address;
3817         pte_t *ptep;
3818         pte_t pte;
3819         struct hstate *h = hstate_vma(vma);
3820         unsigned long pages = 0;
3821
3822         BUG_ON(address >= end);
3823         flush_cache_range(vma, address, end);
3824
3825         mmu_notifier_invalidate_range_start(mm, start, end);
3826         i_mmap_lock_write(vma->vm_file->f_mapping);
3827         for (; address < end; address += huge_page_size(h)) {
3828                 spinlock_t *ptl;
3829                 ptep = huge_pte_offset(mm, address);
3830                 if (!ptep)
3831                         continue;
3832                 ptl = huge_pte_lock(h, mm, ptep);
3833                 if (huge_pmd_unshare(mm, &address, ptep)) {
3834                         pages++;
3835                         spin_unlock(ptl);
3836                         continue;
3837                 }
3838                 pte = huge_ptep_get(ptep);
3839                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
3840                         spin_unlock(ptl);
3841                         continue;
3842                 }
3843                 if (unlikely(is_hugetlb_entry_migration(pte))) {
3844                         swp_entry_t entry = pte_to_swp_entry(pte);
3845
3846                         if (is_write_migration_entry(entry)) {
3847                                 pte_t newpte;
3848
3849                                 make_migration_entry_read(&entry);
3850                                 newpte = swp_entry_to_pte(entry);
3851                                 set_huge_pte_at(mm, address, ptep, newpte);
3852                                 pages++;
3853                         }
3854                         spin_unlock(ptl);
3855                         continue;
3856                 }
3857                 if (!huge_pte_none(pte)) {
3858                         pte = huge_ptep_get_and_clear(mm, address, ptep);
3859                         pte = pte_mkhuge(huge_pte_modify(pte, newprot));
3860                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
3861                         set_huge_pte_at(mm, address, ptep, pte);
3862                         pages++;
3863                 }
3864                 spin_unlock(ptl);
3865         }
3866         /*
3867          * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
3868          * may have cleared our pud entry and done put_page on the page table:
3869          * once we release i_mmap_rwsem, another task can do the final put_page
3870          * and that page table be reused and filled with junk.
3871          */
3872         flush_tlb_range(vma, start, end);
3873         mmu_notifier_invalidate_range(mm, start, end);
3874         i_mmap_unlock_write(vma->vm_file->f_mapping);
3875         mmu_notifier_invalidate_range_end(mm, start, end);
3876
3877         return pages << h->order;
3878 }
3879
3880 int hugetlb_reserve_pages(struct inode *inode,
3881                                         long from, long to,
3882                                         struct vm_area_struct *vma,
3883                                         vm_flags_t vm_flags)
3884 {
3885         long ret, chg;
3886         struct hstate *h = hstate_inode(inode);
3887         struct hugepage_subpool *spool = subpool_inode(inode);
3888         struct resv_map *resv_map;
3889         long gbl_reserve;
3890
3891         /*
3892          * Only apply hugepage reservation if asked. At fault time, an
3893          * attempt will be made for VM_NORESERVE to allocate a page
3894          * without using reserves
3895          */
3896         if (vm_flags & VM_NORESERVE)
3897                 return 0;
3898
3899         /*
3900          * Shared mappings base their reservation on the number of pages that
3901          * are already allocated on behalf of the file. Private mappings need
3902          * to reserve the full area even if read-only as mprotect() may be
3903          * called to make the mapping read-write. Assume !vma is a shm mapping
3904          */
3905         if (!vma || vma->vm_flags & VM_MAYSHARE) {
3906                 resv_map = inode_resv_map(inode);
3907
3908                 chg = region_chg(resv_map, from, to);
3909
3910         } else {
3911                 resv_map = resv_map_alloc();
3912                 if (!resv_map)
3913                         return -ENOMEM;
3914
3915                 chg = to - from;
3916
3917                 set_vma_resv_map(vma, resv_map);
3918                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3919         }
3920
3921         if (chg < 0) {
3922                 ret = chg;
3923                 goto out_err;
3924         }
3925
3926         /*
3927          * There must be enough pages in the subpool for the mapping. If
3928          * the subpool has a minimum size, there may be some global
3929          * reservations already in place (gbl_reserve).
3930          */
3931         gbl_reserve = hugepage_subpool_get_pages(spool, chg);
3932         if (gbl_reserve < 0) {
3933                 ret = -ENOSPC;
3934                 goto out_err;
3935         }
3936
3937         /*
3938          * Check enough hugepages are available for the reservation.
3939          * Hand the pages back to the subpool if there are not
3940          */
3941         ret = hugetlb_acct_memory(h, gbl_reserve);
3942         if (ret < 0) {
3943                 /* put back original number of pages, chg */
3944                 (void)hugepage_subpool_put_pages(spool, chg);
3945                 goto out_err;
3946         }
3947
3948         /*
3949          * Account for the reservations made. Shared mappings record regions
3950          * that have reservations as they are shared by multiple VMAs.
3951          * When the last VMA disappears, the region map says how much
3952          * the reservation was and the page cache tells how much of
3953          * the reservation was consumed. Private mappings are per-VMA and
3954          * only the consumed reservations are tracked. When the VMA
3955          * disappears, the original reservation is the VMA size and the
3956          * consumed reservations are stored in the map. Hence, nothing
3957          * else has to be done for private mappings here
3958          */
3959         if (!vma || vma->vm_flags & VM_MAYSHARE) {
3960                 long add = region_add(resv_map, from, to);
3961
3962                 if (unlikely(chg > add)) {
3963                         /*
3964                          * pages in this range were added to the reserve
3965                          * map between region_chg and region_add.  This
3966                          * indicates a race with alloc_huge_page.  Adjust
3967                          * the subpool and reserve counts modified above
3968                          * based on the difference.
3969                          */
3970                         long rsv_adjust;
3971
3972                         rsv_adjust = hugepage_subpool_put_pages(spool,
3973                                                                 chg - add);
3974                         hugetlb_acct_memory(h, -rsv_adjust);
3975                 }
3976         }
3977         return 0;
3978 out_err:
3979         if (!vma || vma->vm_flags & VM_MAYSHARE)
3980                 region_abort(resv_map, from, to);
3981         if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3982                 kref_put(&resv_map->refs, resv_map_release);
3983         return ret;
3984 }
3985
3986 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
3987                                                                 long freed)
3988 {
3989         struct hstate *h = hstate_inode(inode);
3990         struct resv_map *resv_map = inode_resv_map(inode);
3991         long chg = 0;
3992         struct hugepage_subpool *spool = subpool_inode(inode);
3993         long gbl_reserve;
3994
3995         if (resv_map) {
3996                 chg = region_del(resv_map, start, end);
3997                 /*
3998                  * region_del() can fail in the rare case where a region
3999                  * must be split and another region descriptor can not be
4000                  * allocated.  If end == LONG_MAX, it will not fail.
4001                  */
4002                 if (chg < 0)
4003                         return chg;
4004         }
4005
4006         spin_lock(&inode->i_lock);
4007         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
4008         spin_unlock(&inode->i_lock);
4009
4010         /*
4011          * If the subpool has a minimum size, the number of global
4012          * reservations to be released may be adjusted.
4013          */
4014         gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
4015         hugetlb_acct_memory(h, -gbl_reserve);
4016
4017         return 0;
4018 }
4019
4020 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
4021 static unsigned long page_table_shareable(struct vm_area_struct *svma,
4022                                 struct vm_area_struct *vma,
4023                                 unsigned long addr, pgoff_t idx)
4024 {
4025         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
4026                                 svma->vm_start;
4027         unsigned long sbase = saddr & PUD_MASK;
4028         unsigned long s_end = sbase + PUD_SIZE;
4029
4030         /* Allow segments to share if only one is marked locked */
4031         unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
4032         unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
4033
4034         /*
4035          * match the virtual addresses, permission and the alignment of the
4036          * page table page.
4037          */
4038         if (pmd_index(addr) != pmd_index(saddr) ||
4039             vm_flags != svm_flags ||
4040             sbase < svma->vm_start || svma->vm_end < s_end)
4041                 return 0;
4042
4043         return saddr;
4044 }
4045
4046 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
4047 {
4048         unsigned long base = addr & PUD_MASK;
4049         unsigned long end = base + PUD_SIZE;
4050
4051         /*
4052          * check on proper vm_flags and page table alignment
4053          */
4054         if (vma->vm_flags & VM_MAYSHARE &&
4055             vma->vm_start <= base && end <= vma->vm_end)
4056                 return true;
4057         return false;
4058 }
4059
4060 /*
4061  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
4062  * and returns the corresponding pte. While this is not necessary for the
4063  * !shared pmd case because we can allocate the pmd later as well, it makes the
4064  * code much cleaner. pmd allocation is essential for the shared case because
4065  * pud has to be populated inside the same i_mmap_rwsem section - otherwise
4066  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
4067  * bad pmd for sharing.
4068  */
4069 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4070 {
4071         struct vm_area_struct *vma = find_vma(mm, addr);
4072         struct address_space *mapping = vma->vm_file->f_mapping;
4073         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
4074                         vma->vm_pgoff;
4075         struct vm_area_struct *svma;
4076         unsigned long saddr;
4077         pte_t *spte = NULL;
4078         pte_t *pte;
4079         spinlock_t *ptl;
4080
4081         if (!vma_shareable(vma, addr))
4082                 return (pte_t *)pmd_alloc(mm, pud, addr);
4083
4084         i_mmap_lock_write(mapping);
4085         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
4086                 if (svma == vma)
4087                         continue;
4088
4089                 saddr = page_table_shareable(svma, vma, addr, idx);
4090                 if (saddr) {
4091                         spte = huge_pte_offset(svma->vm_mm, saddr);
4092                         if (spte) {
4093                                 mm_inc_nr_pmds(mm);
4094                                 get_page(virt_to_page(spte));
4095                                 break;
4096                         }
4097                 }
4098         }
4099
4100         if (!spte)
4101                 goto out;
4102
4103         ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
4104         spin_lock(ptl);
4105         if (pud_none(*pud)) {
4106                 pud_populate(mm, pud,
4107                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
4108         } else {
4109                 put_page(virt_to_page(spte));
4110                 mm_inc_nr_pmds(mm);
4111         }
4112         spin_unlock(ptl);
4113 out:
4114         pte = (pte_t *)pmd_alloc(mm, pud, addr);
4115         i_mmap_unlock_write(mapping);
4116         return pte;
4117 }
4118
4119 /*
4120  * unmap huge page backed by shared pte.
4121  *
4122  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
4123  * indicated by page_count > 1, unmap is achieved by clearing pud and
4124  * decrementing the ref count. If count == 1, the pte page is not shared.
4125  *
4126  * called with page table lock held.
4127  *
4128  * returns: 1 successfully unmapped a shared pte page
4129  *          0 the underlying pte page is not shared, or it is the last user
4130  */
4131 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4132 {
4133         pgd_t *pgd = pgd_offset(mm, *addr);
4134         pud_t *pud = pud_offset(pgd, *addr);
4135
4136         BUG_ON(page_count(virt_to_page(ptep)) == 0);
4137         if (page_count(virt_to_page(ptep)) == 1)
4138                 return 0;
4139
4140         pud_clear(pud);
4141         put_page(virt_to_page(ptep));
4142         mm_dec_nr_pmds(mm);
4143         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
4144         return 1;
4145 }
4146 #define want_pmd_share()        (1)
4147 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4148 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4149 {
4150         return NULL;
4151 }
4152
4153 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4154 {
4155         return 0;
4156 }
4157 #define want_pmd_share()        (0)
4158 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4159
4160 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
4161 pte_t *huge_pte_alloc(struct mm_struct *mm,
4162                         unsigned long addr, unsigned long sz)
4163 {
4164         pgd_t *pgd;
4165         pud_t *pud;
4166         pte_t *pte = NULL;
4167
4168         pgd = pgd_offset(mm, addr);
4169         pud = pud_alloc(mm, pgd, addr);
4170         if (pud) {
4171                 if (sz == PUD_SIZE) {
4172                         pte = (pte_t *)pud;
4173                 } else {
4174                         BUG_ON(sz != PMD_SIZE);
4175                         if (want_pmd_share() && pud_none(*pud))
4176                                 pte = huge_pmd_share(mm, addr, pud);
4177                         else
4178                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4179                 }
4180         }
4181         BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
4182
4183         return pte;
4184 }
4185
4186 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
4187 {
4188         pgd_t *pgd;
4189         pud_t *pud;
4190         pmd_t *pmd = NULL;
4191
4192         pgd = pgd_offset(mm, addr);
4193         if (pgd_present(*pgd)) {
4194                 pud = pud_offset(pgd, addr);
4195                 if (pud_present(*pud)) {
4196                         if (pud_huge(*pud))
4197                                 return (pte_t *)pud;
4198                         pmd = pmd_offset(pud, addr);
4199                 }
4200         }
4201         return (pte_t *) pmd;
4202 }
4203
4204 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
4205
4206 /*
4207  * These functions are overwritable if your architecture needs its own
4208  * behavior.
4209  */
4210 struct page * __weak
4211 follow_huge_addr(struct mm_struct *mm, unsigned long address,
4212                               int write)
4213 {
4214         return ERR_PTR(-EINVAL);
4215 }
4216
4217 struct page * __weak
4218 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
4219                 pmd_t *pmd, int flags)
4220 {
4221         struct page *page = NULL;
4222         spinlock_t *ptl;
4223 retry:
4224         ptl = pmd_lockptr(mm, pmd);
4225         spin_lock(ptl);
4226         /*
4227          * make sure that the address range covered by this pmd is not
4228          * unmapped from other threads.
4229          */
4230         if (!pmd_huge(*pmd))
4231                 goto out;
4232         if (pmd_present(*pmd)) {
4233                 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
4234                 if (flags & FOLL_GET)
4235                         get_page(page);
4236         } else {
4237                 if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) {
4238                         spin_unlock(ptl);
4239                         __migration_entry_wait(mm, (pte_t *)pmd, ptl);
4240                         goto retry;
4241                 }
4242                 /*
4243                  * hwpoisoned entry is treated as no_page_table in
4244                  * follow_page_mask().
4245                  */
4246         }
4247 out:
4248         spin_unlock(ptl);
4249         return page;
4250 }
4251
4252 struct page * __weak
4253 follow_huge_pud(struct mm_struct *mm, unsigned long address,
4254                 pud_t *pud, int flags)
4255 {
4256         if (flags & FOLL_GET)
4257                 return NULL;
4258
4259         return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
4260 }
4261
4262 #ifdef CONFIG_MEMORY_FAILURE
4263
4264 /*
4265  * This function is called from memory failure code.
4266  * Assume the caller holds page lock of the head page.
4267  */
4268 int dequeue_hwpoisoned_huge_page(struct page *hpage)
4269 {
4270         struct hstate *h = page_hstate(hpage);
4271         int nid = page_to_nid(hpage);
4272         int ret = -EBUSY;
4273
4274         spin_lock(&hugetlb_lock);
4275         /*
4276          * Just checking !page_huge_active is not enough, because that could be
4277          * an isolated/hwpoisoned hugepage (which have >0 refcount).
4278          */
4279         if (!page_huge_active(hpage) && !page_count(hpage)) {
4280                 /*
4281                  * Hwpoisoned hugepage isn't linked to activelist or freelist,
4282                  * but dangling hpage->lru can trigger list-debug warnings
4283                  * (this happens when we call unpoison_memory() on it),
4284                  * so let it point to itself with list_del_init().
4285                  */
4286                 list_del_init(&hpage->lru);
4287                 set_page_refcounted(hpage);
4288                 h->free_huge_pages--;
4289                 h->free_huge_pages_node[nid]--;
4290                 ret = 0;
4291         }
4292         spin_unlock(&hugetlb_lock);
4293         return ret;
4294 }
4295 #endif
4296
4297 bool isolate_huge_page(struct page *page, struct list_head *list)
4298 {
4299         bool ret = true;
4300
4301         VM_BUG_ON_PAGE(!PageHead(page), page);
4302         spin_lock(&hugetlb_lock);
4303         if (!page_huge_active(page) || !get_page_unless_zero(page)) {
4304                 ret = false;
4305                 goto unlock;
4306         }
4307         clear_page_huge_active(page);
4308         list_move_tail(&page->lru, list);
4309 unlock:
4310         spin_unlock(&hugetlb_lock);
4311         return ret;
4312 }
4313
4314 void putback_active_hugepage(struct page *page)
4315 {
4316         VM_BUG_ON_PAGE(!PageHead(page), page);
4317         spin_lock(&hugetlb_lock);
4318         set_page_huge_active(page);
4319         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
4320         spin_unlock(&hugetlb_lock);
4321         put_page(page);
4322 }