mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
authorBenjamin Herrenschmidt <benh@kernel.crashing.org>
Wed, 22 Jul 2009 05:44:28 +0000 (15:44 +1000)
committerLinus Torvalds <torvalds@linux-foundation.org>
Mon, 27 Jul 2009 19:10:38 +0000 (12:10 -0700)
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()

Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.

Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.

The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
34 files changed:
arch/alpha/include/asm/tlb.h
arch/arm/include/asm/tlb.h
arch/avr32/include/asm/pgalloc.h
arch/cris/include/asm/pgalloc.h
arch/frv/include/asm/pgalloc.h
arch/frv/include/asm/pgtable.h
arch/ia64/include/asm/pgalloc.h
arch/ia64/include/asm/tlb.h
arch/m32r/include/asm/pgalloc.h
arch/m68k/include/asm/motorola_pgalloc.h
arch/m68k/include/asm/sun3_pgalloc.h
arch/microblaze/include/asm/pgalloc.h
arch/mips/include/asm/pgalloc.h
arch/mn10300/include/asm/pgalloc.h
arch/parisc/include/asm/tlb.h
arch/powerpc/include/asm/pgalloc-32.h
arch/powerpc/include/asm/pgalloc-64.h
arch/powerpc/include/asm/pgalloc.h
arch/powerpc/mm/hugetlbpage.c
arch/s390/include/asm/tlb.h
arch/sh/include/asm/pgalloc.h
arch/sh/include/asm/tlb.h
arch/sparc/include/asm/pgalloc_32.h
arch/sparc/include/asm/tlb_64.h
arch/um/include/asm/pgalloc.h
arch/um/include/asm/tlb.h
arch/x86/include/asm/pgalloc.h
arch/x86/mm/pgtable.c
arch/xtensa/include/asm/tlb.h
include/asm-generic/4level-fixup.h
include/asm-generic/pgtable-nopmd.h
include/asm-generic/pgtable-nopud.h
include/asm-generic/tlb.h
mm/memory.c

index c13636575fbab4fb14c17b75d53058b0f9181e1c..42866759f3fabd022c07c431a01bc9f45f6d91f4 100644 (file)
@@ -9,7 +9,7 @@
 
 #include <asm-generic/tlb.h>
 
-#define __pte_free_tlb(tlb, pte)                       pte_free((tlb)->mm, pte)
-#define __pmd_free_tlb(tlb, pmd)                       pmd_free((tlb)->mm, pmd)
+#define __pte_free_tlb(tlb, pte, address)              pte_free((tlb)->mm, pte)
+#define __pmd_free_tlb(tlb, pmd, address)              pmd_free((tlb)->mm, pmd)
  
 #endif
index 321c83e43a1e7a2cc9d8d770417004a5c206be9a..f41a6f57cd1223aef8d00f450bc1b68365623b19 100644 (file)
@@ -102,8 +102,8 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 }
 
 #define tlb_remove_page(tlb,page)      free_page_and_swap_cache(page)
-#define pte_free_tlb(tlb, ptep)                pte_free((tlb)->mm, ptep)
-#define pmd_free_tlb(tlb, pmdp)                pmd_free((tlb)->mm, pmdp)
+#define pte_free_tlb(tlb, ptep, addr)  pte_free((tlb)->mm, ptep)
+#define pmd_free_tlb(tlb, pmdp, addr)  pmd_free((tlb)->mm, pmdp)
 
 #define tlb_migrate_finish(mm)         do { } while (0)
 
index 64082132394312a4c861abdc6b110f58517a7a40..92ecd8446ef80b6fe06c62bce45fe93a40f344bb 100644 (file)
@@ -83,7 +83,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
        quicklist_free_page(QUICK_PT, NULL, pte);
 }
 
-#define __pte_free_tlb(tlb,pte)                                \
+#define __pte_free_tlb(tlb,pte,addr)                   \
 do {                                                   \
        pgtable_page_dtor(pte);                         \
        tlb_remove_page((tlb), pte);                    \
index a1ba761d0573637ffb79ff4f9d6624d644dad9ce..6da975db112fdabff1125ed0f3ecb38df892caae 100644 (file)
@@ -47,7 +47,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
        __free_page(pte);
 }
 
-#define __pte_free_tlb(tlb,pte)                                \
+#define __pte_free_tlb(tlb,pte,address)                        \
 do {                                                   \
        pgtable_page_dtor(pte);                         \
        tlb_remove_page((tlb), pte);                    \
index 971e6addb0095a4175d86b68715937e918d2f474..416d19a632f228a30f1133df32fc1629fa359731 100644 (file)
@@ -49,7 +49,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
        __free_page(pte);
 }
 
-#define __pte_free_tlb(tlb,pte)                                \
+#define __pte_free_tlb(tlb,pte,address)                        \
 do {                                                   \
        pgtable_page_dtor(pte);                         \
        tlb_remove_page((tlb),(pte));                   \
@@ -62,7 +62,7 @@ do {                                                  \
  */
 #define pmd_alloc_one(mm, addr)                ({ BUG(); ((pmd_t *) 2); })
 #define pmd_free(mm, x)                        do { } while (0)
-#define __pmd_free_tlb(tlb,x)          do { } while (0)
+#define __pmd_free_tlb(tlb,x,a)                do { } while (0)
 
 #endif /* CONFIG_MMU */
 
index 33233011b1c1147ff46d2aaac3522150d72630c7..22c60692b5513c7c6f66d49947a16e574ba31835 100644 (file)
@@ -225,7 +225,7 @@ static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
  */
 #define pud_alloc_one(mm, address)             NULL
 #define pud_free(mm, x)                                do { } while (0)
-#define __pud_free_tlb(tlb, x                do { } while (0)
+#define __pud_free_tlb(tlb, x, address)                do { } while (0)
 
 /*
  * The "pud_xxx()" functions here are trivial for a folded two-level
index b9ac1a6fc21694a55e2771e471ab8cc90ab997cf..96a8d927db2851c9c17d8b48f6b47da66f9dd572 100644 (file)
@@ -48,7 +48,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
 {
        quicklist_free(0, NULL, pud);
 }
-#define __pud_free_tlb(tlb, pud      pud_free((tlb)->mm, pud)
+#define __pud_free_tlb(tlb, pud, address)      pud_free((tlb)->mm, pud)
 #endif /* CONFIG_PGTABLE_4 */
 
 static inline void
@@ -67,7 +67,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
        quicklist_free(0, NULL, pmd);
 }
 
-#define __pmd_free_tlb(tlb, pmd      pmd_free((tlb)->mm, pmd)
+#define __pmd_free_tlb(tlb, pmd, address)      pmd_free((tlb)->mm, pmd)
 
 static inline void
 pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, pgtable_t pte)
@@ -117,6 +117,6 @@ static inline void check_pgt_cache(void)
        quicklist_trim(0, NULL, 25, 16);
 }
 
-#define __pte_free_tlb(tlb, pte      pte_free((tlb)->mm, pte)
+#define __pte_free_tlb(tlb, pte, address)      pte_free((tlb)->mm, pte)
 
 #endif                         /* _ASM_IA64_PGALLOC_H */
index 20d8a39680c212af2e455327fb5782680af8c772..85d965cb19a0835ace573551d667053fcc8fa260 100644 (file)
@@ -236,22 +236,22 @@ do {                                                      \
        __tlb_remove_tlb_entry(tlb, ptep, addr);        \
 } while (0)
 
-#define pte_free_tlb(tlb, ptep)                                \
+#define pte_free_tlb(tlb, ptep, address)               \
 do {                                                   \
        tlb->need_flush = 1;                            \
-       __pte_free_tlb(tlb, ptep);                      \
+       __pte_free_tlb(tlb, ptep, address);             \
 } while (0)
 
-#define pmd_free_tlb(tlb, ptep)                                \
+#define pmd_free_tlb(tlb, ptep, address)               \
 do {                                                   \
        tlb->need_flush = 1;                            \
-       __pmd_free_tlb(tlb, ptep);                      \
+       __pmd_free_tlb(tlb, ptep, address);             \
 } while (0)
 
-#define pud_free_tlb(tlb, pudp)                                \
+#define pud_free_tlb(tlb, pudp, address)               \
 do {                                                   \
        tlb->need_flush = 1;                            \
-       __pud_free_tlb(tlb, pudp);                      \
+       __pud_free_tlb(tlb, pudp, address);             \
 } while (0)
 
 #endif /* _ASM_IA64_TLB_H */
index f11a2b909cdbf8ab3bc1a11195b0160173c2a58b..0fc7361989797d9a8c6fb6cc2a823367f75236e0 100644 (file)
@@ -58,7 +58,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
        __free_page(pte);
 }
 
-#define __pte_free_tlb(tlb, pte)       pte_free((tlb)->mm, (pte))
+#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, (pte))
 
 /*
  * allocating and freeing a pmd is trivial: the 1-entry pmd is
@@ -68,7 +68,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
 
 #define pmd_alloc_one(mm, addr)                ({ BUG(); ((pmd_t *)2); })
 #define pmd_free(mm, x)                        do { } while (0)
-#define __pmd_free_tlb(tlb, x)         do { } while (0)
+#define __pmd_free_tlb(tlb, x, addr)   do { } while (0)
 #define pgd_populate(mm, pmd, pte)     BUG()
 
 #define check_pgt_cache()      do { } while (0)
index d08bf6261df88889f0e39091f8356580920208bb..15ee4c74a9f0621c6ca4d45380e6267475edf8e5 100644 (file)
@@ -54,7 +54,8 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t page)
        __free_page(page);
 }
 
-static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page)
+static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page,
+                                 unsigned long address)
 {
        pgtable_page_dtor(page);
        cache_page(kmap(page));
@@ -73,7 +74,8 @@ static inline int pmd_free(struct mm_struct *mm, pmd_t *pmd)
        return free_pointer_table(pmd);
 }
 
-static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
+static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
+                                unsigned long address)
 {
        return free_pointer_table(pmd);
 }
index d4c83f14381652402d35f4b3cb5fb5967f97a6de..48d80d5a666f80b64d5b62e318197b1752e29733 100644 (file)
@@ -32,7 +32,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t page)
         __free_page(page);
 }
 
-#define __pte_free_tlb(tlb,pte)                                \
+#define __pte_free_tlb(tlb,pte,addr)                   \
 do {                                                   \
        pgtable_page_dtor(pte);                         \
        tlb_remove_page((tlb), pte);                    \
@@ -80,7 +80,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page
  * inside the pgd, so has no extra memory associated with it.
  */
 #define pmd_free(mm, x)                        do { } while (0)
-#define __pmd_free_tlb(tlb, x)         do { } while (0)
+#define __pmd_free_tlb(tlb, x, addr)   do { } while (0)
 
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
index 59a757e46ba552ad57736103ea42c3378e83af90..b0131da1387bb131cd9e7666519eedb9591302e3 100644 (file)
@@ -180,7 +180,7 @@ extern inline void pte_free(struct mm_struct *mm, struct page *ptepage)
        __free_page(ptepage);
 }
 
-#define __pte_free_tlb(tlb, pte)       pte_free((tlb)->mm, (pte))
+#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, (pte))
 
 #define pmd_populate(mm, pmd, pte)     (pmd_val(*(pmd)) = page_address(pte))
 
@@ -193,7 +193,7 @@ extern inline void pte_free(struct mm_struct *mm, struct page *ptepage)
  */
 #define pmd_alloc_one(mm, address)     ({ BUG(); ((pmd_t *)2); })
 /*#define pmd_free(mm, x)                      do { } while (0)*/
-#define __pmd_free_tlb(tlb, x)         do { } while (0)
+#define __pmd_free_tlb(tlb, x, addr)   do { } while (0)
 #define pgd_populate(mm, pmd, pte)     BUG()
 
 extern int do_check_pgt_cache(int, int);
index 1275831dda29b65ae90d2046f781a747cb4a3df0..f705735feefcb063eb31d7d35e89e430da081fda 100644 (file)
@@ -98,7 +98,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
        __free_pages(pte, PTE_ORDER);
 }
 
-#define __pte_free_tlb(tlb,pte)                                \
+#define __pte_free_tlb(tlb,pte,address)                        \
 do {                                                   \
        pgtable_page_dtor(pte);                         \
        tlb_remove_page((tlb), pte);                    \
@@ -111,7 +111,7 @@ do {                                                        \
  * inside the pgd, so has no extra memory associated with it.
  */
 #define pmd_free(mm, x)                        do { } while (0)
-#define __pmd_free_tlb(tlb, x)         do { } while (0)
+#define __pmd_free_tlb(tlb, x, addr)   do { } while (0)
 
 #endif
 
@@ -132,7 +132,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
        free_pages((unsigned long)pmd, PMD_ORDER);
 }
 
-#define __pmd_free_tlb(tlb, x) pmd_free((tlb)->mm, x)
+#define __pmd_free_tlb(tlb, x, addr)   pmd_free((tlb)->mm, x)
 
 #endif
 
index ec057e1bd4cf3e72538d4b8c5ae3a83b4c887694..a19f11327cd87ce9dd2c578c1ecf25bf655d083f 100644 (file)
@@ -51,6 +51,6 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte)
 }
 
 
-#define __pte_free_tlb(tlb, pte) tlb_remove_page((tlb), (pte))
+#define __pte_free_tlb(tlb, pte, addr) tlb_remove_page((tlb), (pte))
 
 #endif /* _ASM_PGALLOC_H */
index 383b1db310ee71e18789112587f9ba3e818bdd25..07924903989eed2a8ce2d93cb349332918e1db6e 100644 (file)
@@ -21,7 +21,7 @@ do {  if (!(tlb)->fullmm)     \
 
 #include <asm-generic/tlb.h>
 
-#define __pmd_free_tlb(tlb, pmd)       pmd_free((tlb)->mm, pmd)
-#define __pte_free_tlb(tlb, pte)       pte_free((tlb)->mm, pte)
+#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd)
+#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte)
 
 #endif
index 0815eb40acae214b8947a4da26d565bfaf4fb8a7..c9500d666a1de37687346d7b7cfae5dd2f81ac7a 100644 (file)
@@ -16,7 +16,7 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
  */
 /* #define pmd_alloc_one(mm,address)       ({ BUG(); ((pmd_t *)2); }) */
 #define pmd_free(mm, x)                do { } while (0)
-#define __pmd_free_tlb(tlb,x)          do { } while (0)
+#define __pmd_free_tlb(tlb,x,a)                do { } while (0)
 /* #define pgd_populate(mm, pmd, pte)      BUG() */
 
 #ifndef CONFIG_BOOKE
index afda2bdd860f8bc8eb20dbb915701fface743ce6..e6f069c4f713a7b2c7ce76e3da593693d2fc1d79 100644 (file)
@@ -118,11 +118,11 @@ static inline void pgtable_free(pgtable_free_t pgf)
                kmem_cache_free(pgtable_cache[cachenum], p);
 }
 
-#define __pmd_free_tlb(tlb, pmd)       \
+#define __pmd_free_tlb(tlb, pmd,addr)                \
        pgtable_free_tlb(tlb, pgtable_free_cache(pmd, \
                PMD_CACHE_NUM, PMD_TABLE_SIZE-1))
 #ifndef CONFIG_PPC_64K_PAGES
-#define __pud_free_tlb(tlb, pud)       \
+#define __pud_free_tlb(tlb, pud, addr)               \
        pgtable_free_tlb(tlb, pgtable_free_cache(pud, \
                PUD_CACHE_NUM, PUD_TABLE_SIZE-1))
 #endif /* CONFIG_PPC_64K_PAGES */
index 5d8480265a77561bfa48cb2b82eb2fca9b4a4fd2..1730e5e298d61b07df858de4e32078c7ff4201e4 100644 (file)
@@ -38,14 +38,14 @@ static inline pgtable_free_t pgtable_free_cache(void *p, int cachenum,
 extern void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf);
 
 #ifdef CONFIG_SMP
-#define __pte_free_tlb(tlb,ptepage)    \
+#define __pte_free_tlb(tlb,ptepage,address)            \
 do { \
        pgtable_page_dtor(ptepage); \
        pgtable_free_tlb(tlb, pgtable_free_cache(page_address(ptepage), \
-               PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)); \
+                                       PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)); \
 } while (0)
 #else
-#define __pte_free_tlb(tlb, pte      pte_free((tlb)->mm, (pte))
+#define __pte_free_tlb(tlb, pte, address)      pte_free((tlb)->mm, (pte))
 #endif
 
 
index 9920d6a7cf290cc02339da13d337fa6eddbf021a..c46ef2ffa3d95673febbbe9be0d167a5879ec04f 100644 (file)
@@ -305,7 +305,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
 
        pmd = pmd_offset(pud, start);
        pud_clear(pud);
-       pmd_free_tlb(tlb, pmd);
+       pmd_free_tlb(tlb, pmd, start);
 }
 
 static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
@@ -348,7 +348,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
 
        pud = pud_offset(pgd, start);
        pgd_clear(pgd);
-       pud_free_tlb(tlb, pud);
+       pud_free_tlb(tlb, pud, start);
 }
 
 /*
index 3d8a96d39d9d6a9a999c0f493911af00e204b605..81150b0536890b7d664052a33f22d397bb05b852 100644 (file)
@@ -96,7 +96,8 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
  * pte_free_tlb frees a pte table and clears the CRSTE for the
  * page table from the tlb.
  */
-static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte)
+static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
+                               unsigned long address)
 {
        if (!tlb->fullmm) {
                tlb->array[tlb->nr_ptes++] = pte;
@@ -113,7 +114,8 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte)
  * as the pgd. pmd_free_tlb checks the asce_limit against 2GB
  * to avoid the double free of the pmd in this case.
  */
-static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
+static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
+                               unsigned long address)
 {
 #ifdef __s390x__
        if (tlb->mm->context.asce_limit <= (1UL << 31))
@@ -134,7 +136,8 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
  * as the pgd. pud_free_tlb checks the asce_limit against 4TB
  * to avoid the double free of the pud in this case.
  */
-static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
+static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
+                               unsigned long address)
 {
 #ifdef __s390x__
        if (tlb->mm->context.asce_limit <= (1UL << 42))
index 84dd2db7104c74f4f57eb4f1c7d0bfc1f4159869..89a482750a5b0f267f6360cfb1cf259db6bde269 100644 (file)
@@ -73,7 +73,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
        quicklist_free_page(QUICK_PT, NULL, pte);
 }
 
-#define __pte_free_tlb(tlb,pte)                                \
+#define __pte_free_tlb(tlb,pte,addr)                   \
 do {                                                   \
        pgtable_page_dtor(pte);                         \
        tlb_remove_page((tlb), (pte));                  \
@@ -85,7 +85,7 @@ do {                                                  \
  */
 
 #define pmd_free(mm, x)                        do { } while (0)
-#define __pmd_free_tlb(tlb,x)          do { } while (0)
+#define __pmd_free_tlb(tlb,x,addr)     do { } while (0)
 
 static inline void check_pgt_cache(void)
 {
index 9c16f737074afb62b552170227fdf4571c66912d..da8fe7ab87283904adac91d7fae1a7bea65af497 100644 (file)
@@ -91,9 +91,9 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 }
 
 #define tlb_remove_page(tlb,page)      free_page_and_swap_cache(page)
-#define pte_free_tlb(tlb, ptep)                pte_free((tlb)->mm, ptep)
-#define pmd_free_tlb(tlb, pmdp)                pmd_free((tlb)->mm, pmdp)
-#define pud_free_tlb(tlb, pudp)                pud_free((tlb)->mm, pudp)
+#define pte_free_tlb(tlb, ptep, addr)  pte_free((tlb)->mm, ptep)
+#define pmd_free_tlb(tlb, pmdp, addr)  pmd_free((tlb)->mm, pmdp)
+#define pud_free_tlb(tlb, pudp, addr)  pud_free((tlb)->mm, pudp)
 
 #define tlb_migrate_finish(mm)         do { } while (0)
 
index 681582d2696911ce7262bfa821dc1dbff5f41d94..ca2b34456c4b00c5e5854f630286e0cfe192cc96 100644 (file)
@@ -44,8 +44,8 @@ BTFIXUPDEF_CALL(pmd_t *, pmd_alloc_one, struct mm_struct *, unsigned long)
 BTFIXUPDEF_CALL(void, free_pmd_fast, pmd_t *)
 #define free_pmd_fast(pmd)     BTFIXUP_CALL(free_pmd_fast)(pmd)
 
-#define pmd_free(mm, pmd)      free_pmd_fast(pmd)
-#define __pmd_free_tlb(tlb, pmd) pmd_free((tlb)->mm, pmd)
+#define pmd_free(mm, pmd)              free_pmd_fast(pmd)
+#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd)
 
 BTFIXUPDEF_CALL(void, pmd_populate, pmd_t *, struct page *)
 #define pmd_populate(MM, PMD, PTE)        BTFIXUP_CALL(pmd_populate)(PMD, PTE)
@@ -62,7 +62,7 @@ BTFIXUPDEF_CALL(void, free_pte_fast, pte_t *)
 #define pte_free_kernel(mm, pte)       BTFIXUP_CALL(free_pte_fast)(pte)
 
 BTFIXUPDEF_CALL(void, pte_free, pgtable_t )
-#define pte_free(mm, pte)      BTFIXUP_CALL(pte_free)(pte)
-#define __pte_free_tlb(tlb, pte)       pte_free((tlb)->mm, pte)
+#define pte_free(mm, pte)              BTFIXUP_CALL(pte_free)(pte)
+#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte)
 
 #endif /* _SPARC_PGALLOC_H */
index ee38e731bfa635587afd73bac991fa69fad8ebec..dca406b9b6fc5b9b6400732d6abe3a9f1e9e528d 100644 (file)
@@ -100,9 +100,9 @@ static inline void tlb_remove_page(struct mmu_gather *mp, struct page *page)
 }
 
 #define tlb_remove_tlb_entry(mp,ptep,addr) do { } while (0)
-#define pte_free_tlb(mp, ptepage) pte_free((mp)->mm, ptepage)
-#define pmd_free_tlb(mp, pmdp) pmd_free((mp)->mm, pmdp)
-#define pud_free_tlb(tlb,pudp) __pud_free_tlb(tlb,pudp)
+#define pte_free_tlb(mp, ptepage, addr) pte_free((mp)->mm, ptepage)
+#define pmd_free_tlb(mp, pmdp, addr) pmd_free((mp)->mm, pmdp)
+#define pud_free_tlb(tlb,pudp, addr) __pud_free_tlb(tlb,pudp,addr)
 
 #define tlb_migrate_finish(mm) do { } while (0)
 #define tlb_start_vma(tlb, vma) do { } while (0)
index 718984359f8c8876d84a2a902fa0b6fa6b4cb933..32c8ce4e15153ee6d42fb72fd7579df0400f9e69 100644 (file)
@@ -40,7 +40,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
        __free_page(pte);
 }
 
-#define __pte_free_tlb(tlb,pte)                                \
+#define __pte_free_tlb(tlb,pte, address)               \
 do {                                                   \
        pgtable_page_dtor(pte);                         \
        tlb_remove_page((tlb),(pte));                   \
@@ -53,7 +53,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
        free_page((unsigned long)pmd);
 }
 
-#define __pmd_free_tlb(tlb,x)   tlb_remove_page((tlb),virt_to_page(x))
+#define __pmd_free_tlb(tlb,x, address)   tlb_remove_page((tlb),virt_to_page(x))
 #endif
 
 #define check_pgt_cache()      do { } while (0)
index 5240fa1c5e0860debec1fcf56daf1e3db30970ec..660caedac9eb70bb3314beb5b1e87a3d3e876bef 100644 (file)
@@ -116,11 +116,11 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
                __tlb_remove_tlb_entry(tlb, ptep, address);     \
        } while (0)
 
-#define pte_free_tlb(tlb, ptep) __pte_free_tlb(tlb, ptep)
+#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
 
-#define pud_free_tlb(tlb, pudp) __pud_free_tlb(tlb, pudp)
+#define pud_free_tlb(tlb, pudp, addr) __pud_free_tlb(tlb, pudp, addr)
 
-#define pmd_free_tlb(tlb, pmdp) __pmd_free_tlb(tlb, pmdp)
+#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
 
 #define tlb_migrate_finish(mm) do {} while (0)
 
index dd14c54ac718f465ebccf0b486b421f3d3937c6b..0e8c2a0fd9222d4b75793fb664591de22461e7d7 100644 (file)
@@ -46,7 +46,13 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte)
        __free_page(pte);
 }
 
-extern void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte);
+extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);
+
+static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte,
+                                 unsigned long address)
+{
+       ___pte_free_tlb(tlb, pte);
+}
 
 static inline void pmd_populate_kernel(struct mm_struct *mm,
                                       pmd_t *pmd, pte_t *pte)
@@ -78,7 +84,13 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
        free_page((unsigned long)pmd);
 }
 
-extern void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
+extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
+
+static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
+                                 unsigned long adddress)
+{
+       ___pmd_free_tlb(tlb, pmd);
+}
 
 #ifdef CONFIG_X86_PAE
 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
@@ -108,7 +120,14 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
        free_page((unsigned long)pud);
 }
 
-extern void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud);
+extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud);
+
+static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
+                                 unsigned long address)
+{
+       ___pud_free_tlb(tlb, pud);
+}
+
 #endif /* PAGETABLE_LEVELS > 3 */
 #endif /* PAGETABLE_LEVELS > 2 */
 
index 8e43bdd45456017cd431c2cf9b4678c5b7e9f65d..af8f9650058cfcf59ab46476ac174ec86e41de49 100644 (file)
@@ -25,7 +25,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
        return pte;
 }
 
-void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
+void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
 {
        pgtable_page_dtor(pte);
        paravirt_release_pte(page_to_pfn(pte));
@@ -33,14 +33,14 @@ void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
 }
 
 #if PAGETABLE_LEVELS > 2
-void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
+void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
 {
        paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
        tlb_remove_page(tlb, virt_to_page(pmd));
 }
 
 #if PAGETABLE_LEVELS > 3
-void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
+void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
 {
        paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
        tlb_remove_page(tlb, virt_to_page(pud));
index 31c220faca02ac9cbb3695f735e0221bdc32104a..0d766f9c1083a59cd4a073cb5da0dfc640a06415 100644 (file)
@@ -42,6 +42,6 @@
 
 #include <asm-generic/tlb.h>
 
-#define __pte_free_tlb(tlb, pte)               pte_free((tlb)->mm, pte)
+#define __pte_free_tlb(tlb, pte, address)      pte_free((tlb)->mm, pte)
 
 #endif /* _XTENSA_TLB_H */
index 9d40e879f99ea968e18e07f142519c5839392003..77ff547730af53f8da5850f05ee70866b720f4b9 100644 (file)
@@ -27,9 +27,9 @@
 #define pud_page_vaddr(pud)            pgd_page_vaddr(pud)
 
 #undef pud_free_tlb
-#define pud_free_tlb(tlb, x)            do { } while (0)
+#define pud_free_tlb(tlb, x, addr)     do { } while (0)
 #define pud_free(mm, x)                        do { } while (0)
-#define __pud_free_tlb(tlb, x)         do { } while (0)
+#define __pud_free_tlb(tlb, x, addr)   do { } while (0)
 
 #undef  pud_addr_end
 #define pud_addr_end(addr, end)                (end)
index a7cdc48e8b78703156456f8eb2472251e66c8d78..725612b793ce1d55462de9ab2158a795ab3b4faa 100644 (file)
@@ -59,7 +59,7 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address)
 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
 {
 }
-#define __pmd_free_tlb(tlb, x)                 do { } while (0)
+#define __pmd_free_tlb(tlb, x, a)              do { } while (0)
 
 #undef  pmd_addr_end
 #define pmd_addr_end(addr, end)                        (end)
index 87cf449a6df380e6be3a11fd24648ef3aefe588b..810431d8351b16c14c3d1954ddc2890866c41658 100644 (file)
@@ -52,7 +52,7 @@ static inline pud_t * pud_offset(pgd_t * pgd, unsigned long address)
  */
 #define pud_alloc_one(mm, address)             NULL
 #define pud_free(mm, x)                                do { } while (0)
-#define __pud_free_tlb(tlb, x)                 do { } while (0)
+#define __pud_free_tlb(tlb, x, a)              do { } while (0)
 
 #undef  pud_addr_end
 #define pud_addr_end(addr, end)                        (end)
index f490e43a90b90a09e1cfe78a9f1215b66d3df74e..e43f9766259f301b2087e83d70a1e563e568edbd 100644 (file)
@@ -123,24 +123,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
                __tlb_remove_tlb_entry(tlb, ptep, address);     \
        } while (0)
 
-#define pte_free_tlb(tlb, ptep)                                        \
+#define pte_free_tlb(tlb, ptep, address)                       \
        do {                                                    \
                tlb->need_flush = 1;                            \
-               __pte_free_tlb(tlb, ptep);                      \
+               __pte_free_tlb(tlb, ptep, address);             \
        } while (0)
 
 #ifndef __ARCH_HAS_4LEVEL_HACK
-#define pud_free_tlb(tlb, pudp)                                        \
+#define pud_free_tlb(tlb, pudp, address)                       \
        do {                                                    \
                tlb->need_flush = 1;                            \
-               __pud_free_tlb(tlb, pudp);                      \
+               __pud_free_tlb(tlb, pudp, address);             \
        } while (0)
 #endif
 
-#define pmd_free_tlb(tlb, pmdp)                                        \
+#define pmd_free_tlb(tlb, pmdp, address)                       \
        do {                                                    \
                tlb->need_flush = 1;                            \
-               __pmd_free_tlb(tlb, pmdp);                      \
+               __pmd_free_tlb(tlb, pmdp, address);             \
        } while (0)
 
 #define tlb_migrate_finish(mm) do {} while (0)
index 65216194eb8daec69843ef672d73725eeb37e0ca..aede2ce3aba4fdf1159946cffc7b6acaf8b534d3 100644 (file)
@@ -135,11 +135,12 @@ void pmd_clear_bad(pmd_t *pmd)
  * Note: this doesn't free the actual pages themselves. That
  * has been handled earlier when unmapping all the memory regions.
  */
-static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd)
+static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
+                          unsigned long addr)
 {
        pgtable_t token = pmd_pgtable(*pmd);
        pmd_clear(pmd);
-       pte_free_tlb(tlb, token);
+       pte_free_tlb(tlb, token, addr);
        tlb->mm->nr_ptes--;
 }
 
@@ -157,7 +158,7 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
                next = pmd_addr_end(addr, end);
                if (pmd_none_or_clear_bad(pmd))
                        continue;
-               free_pte_range(tlb, pmd);
+               free_pte_range(tlb, pmd, addr);
        } while (pmd++, addr = next, addr != end);
 
        start &= PUD_MASK;
@@ -173,7 +174,7 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
 
        pmd = pmd_offset(pud, start);
        pud_clear(pud);
-       pmd_free_tlb(tlb, pmd);
+       pmd_free_tlb(tlb, pmd, start);
 }
 
 static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
@@ -206,7 +207,7 @@ static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
 
        pud = pud_offset(pgd, start);
        pgd_clear(pgd);
-       pud_free_tlb(tlb, pud);
+       pud_free_tlb(tlb, pud, start);
 }
 
 /*