Bluetooth: Add support of Toshiba Broadcom based devices
[firefly-linux-kernel-4.4.55.git] / mm / mprotect.c
1 /*
2  *  mm/mprotect.c
3  *
4  *  (C) Copyright 1994 Linus Torvalds
5  *  (C) Copyright 2002 Christoph Hellwig
6  *
7  *  Address space accounting code       <alan@lxorguk.ukuu.org.uk>
8  *  (C) Copyright 2002 Red Hat Inc, All Rights Reserved
9  */
10
11 #include <linux/mm.h>
12 #include <linux/hugetlb.h>
13 #include <linux/shm.h>
14 #include <linux/mman.h>
15 #include <linux/fs.h>
16 #include <linux/highmem.h>
17 #include <linux/security.h>
18 #include <linux/mempolicy.h>
19 #include <linux/personality.h>
20 #include <linux/syscalls.h>
21 #include <linux/swap.h>
22 #include <linux/swapops.h>
23 #include <linux/mmu_notifier.h>
24 #include <linux/migrate.h>
25 #include <linux/perf_event.h>
26 #include <linux/ksm.h>
27 #include <asm/uaccess.h>
28 #include <asm/pgtable.h>
29 #include <asm/cacheflush.h>
30 #include <asm/tlbflush.h>
31
32 #include "internal.h"
33
34 /*
35  * For a prot_numa update we only hold mmap_sem for read so there is a
36  * potential race with faulting where a pmd was temporarily none. This
37  * function checks for a transhuge pmd under the appropriate lock. It
38  * returns a pte if it was successfully locked or NULL if it raced with
39  * a transhuge insertion.
40  */
41 static pte_t *lock_pte_protection(struct vm_area_struct *vma, pmd_t *pmd,
42                         unsigned long addr, int prot_numa, spinlock_t **ptl)
43 {
44         pte_t *pte;
45         spinlock_t *pmdl;
46
47         /* !prot_numa is protected by mmap_sem held for write */
48         if (!prot_numa)
49                 return pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl);
50
51         pmdl = pmd_lock(vma->vm_mm, pmd);
52         if (unlikely(pmd_trans_huge(*pmd) || pmd_none(*pmd))) {
53                 spin_unlock(pmdl);
54                 return NULL;
55         }
56
57         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl);
58         spin_unlock(pmdl);
59         return pte;
60 }
61
62 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
63                 unsigned long addr, unsigned long end, pgprot_t newprot,
64                 int dirty_accountable, int prot_numa)
65 {
66         struct mm_struct *mm = vma->vm_mm;
67         pte_t *pte, oldpte;
68         spinlock_t *ptl;
69         unsigned long pages = 0;
70
71         pte = lock_pte_protection(vma, pmd, addr, prot_numa, &ptl);
72         if (!pte)
73                 return 0;
74
75         arch_enter_lazy_mmu_mode();
76         do {
77                 oldpte = *pte;
78                 if (pte_present(oldpte)) {
79                         pte_t ptent;
80                         bool preserve_write = prot_numa && pte_write(oldpte);
81
82                         /*
83                          * Avoid trapping faults against the zero or KSM
84                          * pages. See similar comment in change_huge_pmd.
85                          */
86                         if (prot_numa) {
87                                 struct page *page;
88
89                                 page = vm_normal_page(vma, addr, oldpte);
90                                 if (!page || PageKsm(page))
91                                         continue;
92
93                                 /* Avoid TLB flush if possible */
94                                 if (pte_protnone(oldpte))
95                                         continue;
96                         }
97
98                         ptent = ptep_modify_prot_start(mm, addr, pte);
99                         ptent = pte_modify(ptent, newprot);
100                         if (preserve_write)
101                                 ptent = pte_mkwrite(ptent);
102
103                         /* Avoid taking write faults for known dirty pages */
104                         if (dirty_accountable && pte_dirty(ptent) &&
105                                         (pte_soft_dirty(ptent) ||
106                                          !(vma->vm_flags & VM_SOFTDIRTY))) {
107                                 ptent = pte_mkwrite(ptent);
108                         }
109                         ptep_modify_prot_commit(mm, addr, pte, ptent);
110                         pages++;
111                 } else if (IS_ENABLED(CONFIG_MIGRATION)) {
112                         swp_entry_t entry = pte_to_swp_entry(oldpte);
113
114                         if (is_write_migration_entry(entry)) {
115                                 pte_t newpte;
116                                 /*
117                                  * A protection check is difficult so
118                                  * just be safe and disable write
119                                  */
120                                 make_migration_entry_read(&entry);
121                                 newpte = swp_entry_to_pte(entry);
122                                 if (pte_swp_soft_dirty(oldpte))
123                                         newpte = pte_swp_mksoft_dirty(newpte);
124                                 set_pte_at(mm, addr, pte, newpte);
125
126                                 pages++;
127                         }
128                 }
129         } while (pte++, addr += PAGE_SIZE, addr != end);
130         arch_leave_lazy_mmu_mode();
131         pte_unmap_unlock(pte - 1, ptl);
132
133         return pages;
134 }
135
136 static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
137                 pud_t *pud, unsigned long addr, unsigned long end,
138                 pgprot_t newprot, int dirty_accountable, int prot_numa)
139 {
140         pmd_t *pmd;
141         struct mm_struct *mm = vma->vm_mm;
142         unsigned long next;
143         unsigned long pages = 0;
144         unsigned long nr_huge_updates = 0;
145         unsigned long mni_start = 0;
146
147         pmd = pmd_offset(pud, addr);
148         do {
149                 unsigned long this_pages;
150
151                 next = pmd_addr_end(addr, end);
152                 if (!pmd_trans_huge(*pmd) && pmd_none_or_clear_bad(pmd))
153                         continue;
154
155                 /* invoke the mmu notifier if the pmd is populated */
156                 if (!mni_start) {
157                         mni_start = addr;
158                         mmu_notifier_invalidate_range_start(mm, mni_start, end);
159                 }
160
161                 if (pmd_trans_huge(*pmd)) {
162                         if (next - addr != HPAGE_PMD_SIZE)
163                                 split_huge_page_pmd(vma, addr, pmd);
164                         else {
165                                 int nr_ptes = change_huge_pmd(vma, pmd, addr,
166                                                 newprot, prot_numa);
167
168                                 if (nr_ptes) {
169                                         if (nr_ptes == HPAGE_PMD_NR) {
170                                                 pages += HPAGE_PMD_NR;
171                                                 nr_huge_updates++;
172                                         }
173
174                                         /* huge pmd was handled */
175                                         continue;
176                                 }
177                         }
178                         /* fall through, the trans huge pmd just split */
179                 }
180                 this_pages = change_pte_range(vma, pmd, addr, next, newprot,
181                                  dirty_accountable, prot_numa);
182                 pages += this_pages;
183         } while (pmd++, addr = next, addr != end);
184
185         if (mni_start)
186                 mmu_notifier_invalidate_range_end(mm, mni_start, end);
187
188         if (nr_huge_updates)
189                 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
190         return pages;
191 }
192
193 static inline unsigned long change_pud_range(struct vm_area_struct *vma,
194                 pgd_t *pgd, unsigned long addr, unsigned long end,
195                 pgprot_t newprot, int dirty_accountable, int prot_numa)
196 {
197         pud_t *pud;
198         unsigned long next;
199         unsigned long pages = 0;
200
201         pud = pud_offset(pgd, addr);
202         do {
203                 next = pud_addr_end(addr, end);
204                 if (pud_none_or_clear_bad(pud))
205                         continue;
206                 pages += change_pmd_range(vma, pud, addr, next, newprot,
207                                  dirty_accountable, prot_numa);
208         } while (pud++, addr = next, addr != end);
209
210         return pages;
211 }
212
213 static unsigned long change_protection_range(struct vm_area_struct *vma,
214                 unsigned long addr, unsigned long end, pgprot_t newprot,
215                 int dirty_accountable, int prot_numa)
216 {
217         struct mm_struct *mm = vma->vm_mm;
218         pgd_t *pgd;
219         unsigned long next;
220         unsigned long start = addr;
221         unsigned long pages = 0;
222
223         BUG_ON(addr >= end);
224         pgd = pgd_offset(mm, addr);
225         flush_cache_range(vma, addr, end);
226         set_tlb_flush_pending(mm);
227         do {
228                 next = pgd_addr_end(addr, end);
229                 if (pgd_none_or_clear_bad(pgd))
230                         continue;
231                 pages += change_pud_range(vma, pgd, addr, next, newprot,
232                                  dirty_accountable, prot_numa);
233         } while (pgd++, addr = next, addr != end);
234
235         /* Only flush the TLB if we actually modified any entries: */
236         if (pages)
237                 flush_tlb_range(vma, start, end);
238         clear_tlb_flush_pending(mm);
239
240         return pages;
241 }
242
243 unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
244                        unsigned long end, pgprot_t newprot,
245                        int dirty_accountable, int prot_numa)
246 {
247         unsigned long pages;
248
249         if (is_vm_hugetlb_page(vma))
250                 pages = hugetlb_change_protection(vma, start, end, newprot);
251         else
252                 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa);
253
254         return pages;
255 }
256
257 int
258 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
259         unsigned long start, unsigned long end, unsigned long newflags)
260 {
261         struct mm_struct *mm = vma->vm_mm;
262         unsigned long oldflags = vma->vm_flags;
263         long nrpages = (end - start) >> PAGE_SHIFT;
264         unsigned long charged = 0;
265         pgoff_t pgoff;
266         int error;
267         int dirty_accountable = 0;
268
269         if (newflags == oldflags) {
270                 *pprev = vma;
271                 return 0;
272         }
273
274         /*
275          * If we make a private mapping writable we increase our commit;
276          * but (without finer accounting) cannot reduce our commit if we
277          * make it unwritable again. hugetlb mapping were accounted for
278          * even if read-only so there is no need to account for them here
279          */
280         if (newflags & VM_WRITE) {
281                 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
282                                                 VM_SHARED|VM_NORESERVE))) {
283                         charged = nrpages;
284                         if (security_vm_enough_memory_mm(mm, charged))
285                                 return -ENOMEM;
286                         newflags |= VM_ACCOUNT;
287                 }
288         }
289
290         /*
291          * First try to merge with previous and/or next vma.
292          */
293         pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
294         *pprev = vma_merge(mm, *pprev, start, end, newflags,
295                            vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
296                            vma->vm_userfaultfd_ctx);
297         if (*pprev) {
298                 vma = *pprev;
299                 goto success;
300         }
301
302         *pprev = vma;
303
304         if (start != vma->vm_start) {
305                 error = split_vma(mm, vma, start, 1);
306                 if (error)
307                         goto fail;
308         }
309
310         if (end != vma->vm_end) {
311                 error = split_vma(mm, vma, end, 0);
312                 if (error)
313                         goto fail;
314         }
315
316 success:
317         /*
318          * vm_flags and vm_page_prot are protected by the mmap_sem
319          * held in write mode.
320          */
321         vma->vm_flags = newflags;
322         dirty_accountable = vma_wants_writenotify(vma);
323         vma_set_page_prot(vma);
324
325         change_protection(vma, start, end, vma->vm_page_prot,
326                           dirty_accountable, 0);
327
328         /*
329          * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
330          * fault on access.
331          */
332         if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED &&
333                         (newflags & VM_WRITE)) {
334                 populate_vma_page_range(vma, start, end, NULL);
335         }
336
337         vm_stat_account(mm, oldflags, vma->vm_file, -nrpages);
338         vm_stat_account(mm, newflags, vma->vm_file, nrpages);
339         perf_event_mmap(vma);
340         return 0;
341
342 fail:
343         vm_unacct_memory(charged);
344         return error;
345 }
346
347 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
348                 unsigned long, prot)
349 {
350         unsigned long vm_flags, nstart, end, tmp, reqprot;
351         struct vm_area_struct *vma, *prev;
352         int error = -EINVAL;
353         const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
354         prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
355         if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
356                 return -EINVAL;
357
358         if (start & ~PAGE_MASK)
359                 return -EINVAL;
360         if (!len)
361                 return 0;
362         len = PAGE_ALIGN(len);
363         end = start + len;
364         if (end <= start)
365                 return -ENOMEM;
366         if (!arch_validate_prot(prot))
367                 return -EINVAL;
368
369         reqprot = prot;
370         /*
371          * Does the application expect PROT_READ to imply PROT_EXEC:
372          */
373         if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
374                 prot |= PROT_EXEC;
375
376         vm_flags = calc_vm_prot_bits(prot);
377
378         down_write(&current->mm->mmap_sem);
379
380         vma = find_vma(current->mm, start);
381         error = -ENOMEM;
382         if (!vma)
383                 goto out;
384         prev = vma->vm_prev;
385         if (unlikely(grows & PROT_GROWSDOWN)) {
386                 if (vma->vm_start >= end)
387                         goto out;
388                 start = vma->vm_start;
389                 error = -EINVAL;
390                 if (!(vma->vm_flags & VM_GROWSDOWN))
391                         goto out;
392         } else {
393                 if (vma->vm_start > start)
394                         goto out;
395                 if (unlikely(grows & PROT_GROWSUP)) {
396                         end = vma->vm_end;
397                         error = -EINVAL;
398                         if (!(vma->vm_flags & VM_GROWSUP))
399                                 goto out;
400                 }
401         }
402         if (start > vma->vm_start)
403                 prev = vma;
404
405         for (nstart = start ; ; ) {
406                 unsigned long newflags;
407
408                 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
409
410                 newflags = vm_flags;
411                 newflags |= (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC));
412
413                 /* newflags >> 4 shift VM_MAY% in place of VM_% */
414                 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) {
415                         error = -EACCES;
416                         goto out;
417                 }
418
419                 error = security_file_mprotect(vma, reqprot, prot);
420                 if (error)
421                         goto out;
422
423                 tmp = vma->vm_end;
424                 if (tmp > end)
425                         tmp = end;
426                 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
427                 if (error)
428                         goto out;
429                 nstart = tmp;
430
431                 if (nstart < prev->vm_end)
432                         nstart = prev->vm_end;
433                 if (nstart >= end)
434                         goto out;
435
436                 vma = prev->vm_next;
437                 if (!vma || vma->vm_start != nstart) {
438                         error = -ENOMEM;
439                         goto out;
440                 }
441         }
442 out:
443         up_write(&current->mm->mmap_sem);
444         return error;
445 }