mm,thp: introduce flush_pmd_tlb_range

[deliverable/linux.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index b16279cbd91df6abe691c56af537e52cc8d8c34a..e25eb3d2081a04f68a3c496ee4665c9e712e24f3 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -25,6 +25,7 @@
 #include <linux/migrate.h>
 #include <linux/hashtable.h>
 #include <linux/userfaultfd_k.h>
+#include <linux/page_idle.h>
 
 #include <asm/tlb.h>
 #include <asm/pgalloc.h>
@@ -1757,6 +1758,11 @@ static void __split_huge_page_refcount(struct page *page,
                /* clear PageTail before overwriting first_page */
                smp_wmb();
 
+               if (page_is_young(page))
+                       set_page_young(page_tail);
+               if (page_is_idle(page))
+                       set_page_idle(page_tail);
+
                /*
                 * __split_huge_page_splitting() already set the
                 * splitting bit in all pmd that could map this
@@ -1874,7 +1880,7 @@ static int __split_huge_page_map(struct page *page,
                 * here). But it is generally safer to never allow
                 * small and huge TLB entries for the same virtual
                 * address to be loaded simultaneously. So instead of
-                * doing "pmd_populate(); flush_tlb_range();" we first
+                * doing "pmd_populate(); flush_pmd_tlb_range();" we first
                 * mark the current pmd notpresent (atomically because
                 * here the pmd_trans_huge and pmd_trans_splitting
                 * must remain set at all times on the pmd until the
@@ -2262,7 +2268,8 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
                VM_BUG_ON_PAGE(PageLRU(page), page);
 
                /* If there is no mapped pte young don't collapse the page */
-               if (pte_young(pteval) || PageReferenced(page) ||
+               if (pte_young(pteval) ||
+                   page_is_young(page) || PageReferenced(page) ||
                    mmu_notifier_test_young(vma->vm_mm, address))
                        referenced = true;
        }
@@ -2693,7 +2700,8 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
                 */
                if (page_count(page) != 1 + !!PageSwapCache(page))
                        goto out_unmap;
-               if (pte_young(pteval) || PageReferenced(page) ||
+               if (pte_young(pteval) ||
+                   page_is_young(page) || PageReferenced(page) ||
                    mmu_notifier_test_young(vma->vm_mm, address))
                        referenced = true;
        }