nfs: don't call blocking operations while !TASK_RUNNING

[firefly-linux-kernel-4.4.55.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 64f8aadb0595d5785c6c60bb6c6fe59f29c2c687..d21c9ef0943c3b6173c3f71027e0f7ca6db92f0c 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1166,7 +1166,7 @@ alloc:
 
        if (unlikely(!new_page)) {
                count_vm_event(THP_FAULT_FALLBACK);
-               if (is_huge_zero_pmd(orig_pmd)) {
+               if (!page) {
                        ret = do_huge_pmd_wp_zero_page_fallback(mm, vma,
                                        address, pmd, orig_pmd, haddr);
                } else {
@@ -1190,7 +1190,7 @@ alloc:
                goto out;
        }
 
-       if (is_huge_zero_pmd(orig_pmd))
+       if (!page)
                clear_huge_page(new_page, haddr, HPAGE_PMD_NR);
        else
                copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
@@ -1215,7 +1215,7 @@ alloc:
                page_add_new_anon_rmap(new_page, vma, haddr);
                set_pmd_at(mm, haddr, pmd, entry);
                update_mmu_cache_pmd(vma, address, pmd);
-               if (is_huge_zero_pmd(orig_pmd)) {
+               if (!page) {
                        add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
                        put_huge_zero_page();
                } else {
@@ -1291,18 +1291,19 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
        struct anon_vma *anon_vma = NULL;
        struct page *page;
        unsigned long haddr = addr & HPAGE_PMD_MASK;
+       int page_nid = -1, this_nid = numa_node_id();
        int target_nid;
-       int current_nid = -1;
-       bool migrated, page_locked;
+       bool page_locked;
+       bool migrated = false;
 
        spin_lock(&mm->page_table_lock);
        if (unlikely(!pmd_same(pmd, *pmdp)))
                goto out_unlock;
 
        page = pmd_page(pmd);
-       current_nid = page_to_nid(page);
+       page_nid = page_to_nid(page);
        count_vm_numa_event(NUMA_HINT_FAULTS);
-       if (current_nid == numa_node_id())
+       if (page_nid == this_nid)
                count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
 
        /*
@@ -1313,24 +1314,25 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
        target_nid = mpol_misplaced(page, vma, haddr);
        if (target_nid == -1) {
                /* If the page was locked, there are no parallel migrations */
-               if (page_locked) {
-                       unlock_page(page);
+               if (page_locked)
                        goto clear_pmdnuma;
-               }
 
-               /* Otherwise wait for potential migrations and retry fault */
+               /*
+                * Otherwise wait for potential migrations and retry. We do
+                * relock and check_same as the page may no longer be mapped.
+                * As the fault is being retried, do not account for it.
+                */
                spin_unlock(&mm->page_table_lock);
                wait_on_page_locked(page);
+               page_nid = -1;
                goto out;
        }
 
        /* Page is misplaced, serialise migrations and parallel THP splits */
        get_page(page);
        spin_unlock(&mm->page_table_lock);
-       if (!page_locked) {
+       if (!page_locked)
                lock_page(page);
-               page_locked = true;
-       }
        anon_vma = page_lock_anon_vma_read(page);
 
        /* Confirm the PTE did not while locked */
@@ -1338,33 +1340,42 @@ int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
        if (unlikely(!pmd_same(pmd, *pmdp))) {
                unlock_page(page);
                put_page(page);
+               page_nid = -1;
                goto out_unlock;
        }
 
-       /* Migrate the THP to the requested node */
+       /* Bail if we fail to protect against THP splits for any reason */
+       if (unlikely(!anon_vma)) {
+               put_page(page);
+               page_nid = -1;
+               goto clear_pmdnuma;
+       }
+
+       /*
+        * The page_table_lock above provides a memory barrier
+        * with change_protection_range.
+        */
+       if (mm_tlb_flush_pending(mm))
+               flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
+
+       /*
+        * Migrate the THP to the requested node, returns with page unlocked
+        * and pmd_numa cleared.
+        */
        spin_unlock(&mm->page_table_lock);
        migrated = migrate_misplaced_transhuge_page(mm, vma,
                                pmdp, pmd, addr, page, target_nid);
-       if (!migrated)
-               goto check_same;
-
-       task_numa_fault(target_nid, HPAGE_PMD_NR, true);
-       if (anon_vma)
-               page_unlock_anon_vma_read(anon_vma);
-       return 0;
+       if (migrated)
+               page_nid = target_nid;
 
-check_same:
-       spin_lock(&mm->page_table_lock);
-       if (unlikely(!pmd_same(pmd, *pmdp))) {
-               /* Someone else took our fault */
-               current_nid = -1;
-               goto out_unlock;
-       }
+       goto out;
 clear_pmdnuma:
+       BUG_ON(!PageLocked(page));
        pmd = pmd_mknonnuma(pmd);
        set_pmd_at(mm, haddr, pmdp, pmd);
        VM_BUG_ON(pmd_numa(*pmdp));
        update_mmu_cache_pmd(vma, addr, pmdp);
+       unlock_page(page);
 out_unlock:
        spin_unlock(&mm->page_table_lock);
 
@@ -1372,8 +1383,9 @@ out:
        if (anon_vma)
                page_unlock_anon_vma_read(anon_vma);
 
-       if (current_nid != -1)
-               task_numa_fault(current_nid, HPAGE_PMD_NR, false);
+       if (page_nid != -1)
+               task_numa_fault(page_nid, HPAGE_PMD_NR, migrated);
+
        return 0;
 }
 
@@ -1721,21 +1733,24 @@ static int __split_huge_page_map(struct page *page,
        if (pmd) {
                pgtable = pgtable_trans_huge_withdraw(mm);
                pmd_populate(mm, &_pmd, pgtable);
+               if (pmd_write(*pmd))
+                       BUG_ON(page_mapcount(page) != 1);
 
                haddr = address;
                for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
                        pte_t *pte, entry;
                        BUG_ON(PageCompound(page+i));
+                       /*
+                        * Note that pmd_numa is not transferred deliberately
+                        * to avoid any possibility that pte_numa leaks to
+                        * a PROT_NONE VMA by accident.
+                        */
                        entry = mk_pte(page + i, vma->vm_page_prot);
                        entry = maybe_mkwrite(pte_mkdirty(entry), vma);
                        if (!pmd_write(*pmd))
                                entry = pte_wrprotect(entry);
-                       else
-                               BUG_ON(page_mapcount(page) != 1);
                        if (!pmd_young(*pmd))
                                entry = pte_mkold(entry);
-                       if (pmd_numa(*pmd))
-                               entry = pte_mknuma(entry);
                        pte = pte_offset_map(&_pmd, haddr);
                        BUG_ON(!pte_none(*pte));
                        set_pte_at(mm, haddr, pte, entry);