This converts the generic user string functions to use the batched user
access functions.
It makes a big difference on Skylake, which is the first x86
microarchitecture to implement SMAP. The STAC/CLAC instructions are not
very fast, and doing them for each access inside the loop that copies
strings from user space (which is what the pathname handling does for
every pathname the kernel uses, for example) is very inefficient.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
(cherry picked from commit
9fd4470ff4974c41b1db43c3b355b9085af9c12a)
Signed-off-by: Alex Shi <alex.shi@linaro.org>
unsigned long c, data;
/* Fall back to byte-at-a-time if we get a page fault */
unsigned long c, data;
/* Fall back to byte-at-a-time if we get a page fault */
- if (unlikely(__get_user(c,(unsigned long __user *)(src+res))))
+ if (unlikely(unsafe_get_user(c,(unsigned long __user *)(src+res))))
break;
*(unsigned long *)(dst+res) = c;
if (has_zero(c, &data, &constants)) {
break;
*(unsigned long *)(dst+res) = c;
if (has_zero(c, &data, &constants)) {
- if (unlikely(__get_user(c,src+res)))
+ if (unlikely(unsafe_get_user(c,src+res)))
return -EFAULT;
dst[res] = c;
if (!c)
return -EFAULT;
dst[res] = c;
if (!c)
src_addr = (unsigned long)src;
if (likely(src_addr < max_addr)) {
unsigned long max = max_addr - src_addr;
src_addr = (unsigned long)src;
if (likely(src_addr < max_addr)) {
unsigned long max = max_addr - src_addr;
- return do_strncpy_from_user(dst, src, count, max);
+ long retval;
+
+ user_access_begin();
+ retval = do_strncpy_from_user(dst, src, count, max);
+ user_access_end();
+ return retval;
src -= align;
max += align;
src -= align;
max += align;
- if (unlikely(__get_user(c,(unsigned long __user *)src)))
+ if (unlikely(unsafe_get_user(c,(unsigned long __user *)src)))
return 0;
c |= aligned_byte_mask(align);
return 0;
c |= aligned_byte_mask(align);
if (unlikely(max <= sizeof(unsigned long)))
break;
max -= sizeof(unsigned long);
if (unlikely(max <= sizeof(unsigned long)))
break;
max -= sizeof(unsigned long);
- if (unlikely(__get_user(c,(unsigned long __user *)(src+res))))
+ if (unlikely(unsafe_get_user(c,(unsigned long __user *)(src+res))))
return 0;
}
res -= align;
return 0;
}
res -= align;
src_addr = (unsigned long)str;
if (likely(src_addr < max_addr)) {
unsigned long max = max_addr - src_addr;
src_addr = (unsigned long)str;
if (likely(src_addr < max_addr)) {
unsigned long max = max_addr - src_addr;
- return do_strnlen_user(str, count, max);
+ long retval;
+
+ user_access_begin();
+ retval = do_strnlen_user(str, count, max);
+ user_access_end();
+ return retval;
src_addr = (unsigned long)str;
if (likely(src_addr < max_addr)) {
unsigned long max = max_addr - src_addr;
src_addr = (unsigned long)str;
if (likely(src_addr < max_addr)) {
unsigned long max = max_addr - src_addr;
- return do_strnlen_user(str, ~0ul, max);
+ long retval;
+
+ user_access_begin();
+ retval = do_strnlen_user(str, ~0ul, max);
+ user_access_end();
+ return retval;