arm64: mm: enable RCU fast_gup

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / include / asm / tlb.h

diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 62731ef9749a66faf30a267e1f6d6c2719aee3e3..a82c0c5c8b521dc652fe9c4ca92de19f07018543 100644 (file)
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -23,6 +23,20 @@
 
 #include <asm-generic/tlb.h>
 
+#include <linux/pagemap.h>
+#include <linux/swap.h>
+
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+
+#define tlb_remove_entry(tlb, entry)   tlb_remove_table(tlb, entry)
+static inline void __tlb_remove_table(void *_table)
+{
+       free_page_and_swap_cache((struct page *)_table);
+}
+#else
+#define tlb_remove_entry(tlb, entry)   tlb_remove_page(tlb, entry)
+#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
+
 /*
  * There's three ways the TLB shootdown code is used:
  *  1. Unmapping a range of vmas.  See zap_page_range(), unmap_region().
@@ -88,7 +102,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
 {
        pgtable_page_dtor(pte);
        tlb_add_flush(tlb, addr);
-       tlb_remove_page(tlb, pte);
+       tlb_remove_entry(tlb, pte);
 }
 
 #if CONFIG_ARM64_PGTABLE_LEVELS > 2
@@ -96,7 +110,7 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
                                  unsigned long addr)
 {
        tlb_add_flush(tlb, addr);
-       tlb_remove_page(tlb, virt_to_page(pmdp));
+       tlb_remove_entry(tlb, virt_to_page(pmdp));
 }
 #endif
 
@@ -105,7 +119,7 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
                                  unsigned long addr)
 {
        tlb_add_flush(tlb, addr);
-       tlb_remove_page(tlb, virt_to_page(pudp));
+       tlb_remove_entry(tlb, virt_to_page(pudp));
 }
 #endif