mm: buffered write iterator
authorNick Piggin <npiggin@suse.de>
Tue, 16 Oct 2007 08:24:59 +0000 (01:24 -0700)
committerLinus Torvalds <torvalds@woody.linux-foundation.org>
Tue, 16 Oct 2007 16:42:55 +0000 (09:42 -0700)
Add an iterator data structure to operate over an iovec.  Add usercopy
operators needed by generic_file_buffered_write, and convert that function
over.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
include/linux/fs.h
mm/filemap.c
mm/filemap.h [deleted file]

index 500ffc0e4ac7856f4cd9487794f2cdd9ed35195e..86ce27c725540244d0e2c09303d67b169a8848aa 100644 (file)
@@ -401,6 +401,39 @@ struct page;
 struct address_space;
 struct writeback_control;
 
+struct iov_iter {
+       const struct iovec *iov;
+       unsigned long nr_segs;
+       size_t iov_offset;
+       size_t count;
+};
+
+size_t iov_iter_copy_from_user_atomic(struct page *page,
+               struct iov_iter *i, unsigned long offset, size_t bytes);
+size_t iov_iter_copy_from_user(struct page *page,
+               struct iov_iter *i, unsigned long offset, size_t bytes);
+void iov_iter_advance(struct iov_iter *i, size_t bytes);
+int iov_iter_fault_in_readable(struct iov_iter *i);
+size_t iov_iter_single_seg_count(struct iov_iter *i);
+
+static inline void iov_iter_init(struct iov_iter *i,
+                       const struct iovec *iov, unsigned long nr_segs,
+                       size_t count, size_t written)
+{
+       i->iov = iov;
+       i->nr_segs = nr_segs;
+       i->iov_offset = 0;
+       i->count = count + written;
+
+       iov_iter_advance(i, written);
+}
+
+static inline size_t iov_iter_count(struct iov_iter *i)
+{
+       return i->count;
+}
+
+
 struct address_space_operations {
        int (*writepage)(struct page *page, struct writeback_control *wbc);
        int (*readpage)(struct file *, struct page *);
index 557fd887254f6cb2b891d0fd11a4c09e79283e3f..67a03a0a9aee0f2d39db9f92a3429355a449fb2b 100644 (file)
@@ -30,7 +30,7 @@
 #include <linux/security.h>
 #include <linux/syscalls.h>
 #include <linux/cpuset.h>
-#include "filemap.h"
+#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
 #include "internal.h"
 
 /*
@@ -1635,8 +1635,7 @@ int remove_suid(struct dentry *dentry)
 }
 EXPORT_SYMBOL(remove_suid);
 
-size_t
-__filemap_copy_from_user_iovec_inatomic(char *vaddr,
+static size_t __iovec_copy_from_user_inatomic(char *vaddr,
                        const struct iovec *iov, size_t base, size_t bytes)
 {
        size_t copied = 0, left = 0;
@@ -1658,6 +1657,110 @@ __filemap_copy_from_user_iovec_inatomic(char *vaddr,
        return copied - left;
 }
 
+/*
+ * Copy as much as we can into the page and return the number of bytes which
+ * were sucessfully copied.  If a fault is encountered then return the number of
+ * bytes which were copied.
+ */
+size_t iov_iter_copy_from_user_atomic(struct page *page,
+               struct iov_iter *i, unsigned long offset, size_t bytes)
+{
+       char *kaddr;
+       size_t copied;
+
+       BUG_ON(!in_atomic());
+       kaddr = kmap_atomic(page, KM_USER0);
+       if (likely(i->nr_segs == 1)) {
+               int left;
+               char __user *buf = i->iov->iov_base + i->iov_offset;
+               left = __copy_from_user_inatomic_nocache(kaddr + offset,
+                                                       buf, bytes);
+               copied = bytes - left;
+       } else {
+               copied = __iovec_copy_from_user_inatomic(kaddr + offset,
+                                               i->iov, i->iov_offset, bytes);
+       }
+       kunmap_atomic(kaddr, KM_USER0);
+
+       return copied;
+}
+
+/*
+ * This has the same sideeffects and return value as
+ * iov_iter_copy_from_user_atomic().
+ * The difference is that it attempts to resolve faults.
+ * Page must not be locked.
+ */
+size_t iov_iter_copy_from_user(struct page *page,
+               struct iov_iter *i, unsigned long offset, size_t bytes)
+{
+       char *kaddr;
+       size_t copied;
+
+       kaddr = kmap(page);
+       if (likely(i->nr_segs == 1)) {
+               int left;
+               char __user *buf = i->iov->iov_base + i->iov_offset;
+               left = __copy_from_user_nocache(kaddr + offset, buf, bytes);
+               copied = bytes - left;
+       } else {
+               copied = __iovec_copy_from_user_inatomic(kaddr + offset,
+                                               i->iov, i->iov_offset, bytes);
+       }
+       kunmap(page);
+       return copied;
+}
+
+static void __iov_iter_advance_iov(struct iov_iter *i, size_t bytes)
+{
+       if (likely(i->nr_segs == 1)) {
+               i->iov_offset += bytes;
+       } else {
+               const struct iovec *iov = i->iov;
+               size_t base = i->iov_offset;
+
+               while (bytes) {
+                       int copy = min(bytes, iov->iov_len - base);
+
+                       bytes -= copy;
+                       base += copy;
+                       if (iov->iov_len == base) {
+                               iov++;
+                               base = 0;
+                       }
+               }
+               i->iov = iov;
+               i->iov_offset = base;
+       }
+}
+
+void iov_iter_advance(struct iov_iter *i, size_t bytes)
+{
+       BUG_ON(i->count < bytes);
+
+       __iov_iter_advance_iov(i, bytes);
+       i->count -= bytes;
+}
+
+int iov_iter_fault_in_readable(struct iov_iter *i)
+{
+       size_t seglen = min(i->iov->iov_len - i->iov_offset, i->count);
+       char __user *buf = i->iov->iov_base + i->iov_offset;
+       return fault_in_pages_readable(buf, seglen);
+}
+
+/*
+ * Return the count of just the current iov_iter segment.
+ */
+size_t iov_iter_single_seg_count(struct iov_iter *i)
+{
+       const struct iovec *iov = i->iov;
+       if (i->nr_segs == 1)
+               return i->count;
+       else
+               return min(i->count, iov->iov_len - i->iov_offset);
+}
+
 /*
  * Performs necessary checks before doing a write
  *
@@ -1816,30 +1919,22 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
        const struct address_space_operations *a_ops = mapping->a_ops;
        struct inode    *inode = mapping->host;
        long            status = 0;
-       const struct iovec *cur_iov = iov; /* current iovec */
-       size_t          iov_offset = 0;    /* offset in the current iovec */
-       char __user     *buf;
+       struct iov_iter i;
 
-       /*
-        * handle partial DIO write.  Adjust cur_iov if needed.
-        */
-       filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, written);
+       iov_iter_init(&i, iov, nr_segs, count, written);
 
        do {
                struct page *src_page;
                struct page *page;
                pgoff_t index;          /* Pagecache index for current page */
                unsigned long offset;   /* Offset into pagecache page */
-               unsigned long seglen;   /* Bytes remaining in current iovec */
                unsigned long bytes;    /* Bytes to write to page */
                size_t copied;          /* Bytes copied from user */
 
-               buf = cur_iov->iov_base + iov_offset;
                offset = (pos & (PAGE_CACHE_SIZE - 1));
                index = pos >> PAGE_CACHE_SHIFT;
-               bytes = PAGE_CACHE_SIZE - offset;
-               if (bytes > count)
-                       bytes = count;
+               bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
+                                               iov_iter_count(&i));
 
                /*
                 * a non-NULL src_page indicates that we're doing the
@@ -1847,10 +1942,6 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
                 */
                src_page = NULL;
 
-               seglen = cur_iov->iov_len - iov_offset;
-               if (seglen > bytes)
-                       seglen = bytes;
-
                /*
                 * Bring in the user page that we will copy from _first_.
                 * Otherwise there's a nasty deadlock on copying from the
@@ -1861,7 +1952,7 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
                 * to check that the address is actually valid, when atomic
                 * usercopies are used, below.
                 */
-               if (unlikely(fault_in_pages_readable(buf, seglen))) {
+               if (unlikely(iov_iter_fault_in_readable(&i))) {
                        status = -EFAULT;
                        break;
                }
@@ -1892,8 +1983,8 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
                         * same reason as we can't take a page fault with a
                         * page locked (as explained below).
                         */
-                       copied = filemap_copy_from_user(src_page, offset,
-                                       cur_iov, nr_segs, iov_offset, bytes);
+                       copied = iov_iter_copy_from_user(src_page, &i,
+                                                               offset, bytes);
                        if (unlikely(copied == 0)) {
                                status = -EFAULT;
                                page_cache_release(page);
@@ -1939,8 +2030,8 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
                         * really matter.
                         */
                        pagefault_disable();
-                       copied = filemap_copy_from_user_atomic(page, offset,
-                                       cur_iov, nr_segs, iov_offset, bytes);
+                       copied = iov_iter_copy_from_user_atomic(page, &i,
+                                                               offset, bytes);
                        pagefault_enable();
                } else {
                        void *src, *dst;
@@ -1965,10 +2056,9 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
                if (src_page)
                        page_cache_release(src_page);
 
+               iov_iter_advance(&i, copied);
                written += copied;
-               count -= copied;
                pos += copied;
-               filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, copied);
 
                balance_dirty_pages_ratelimited(mapping);
                cond_resched();
@@ -1992,7 +2082,7 @@ fs_write_aop_error:
                        continue;
                else
                        break;
-       } while (count);
+       } while (iov_iter_count(&i));
        *ppos = pos;
 
        /*
diff --git a/mm/filemap.h b/mm/filemap.h
deleted file mode 100644 (file)
index b500d93..0000000
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- *     linux/mm/filemap.h
- *
- * Copyright (C) 1994-1999  Linus Torvalds
- */
-
-#ifndef __FILEMAP_H
-#define __FILEMAP_H
-
-#include <linux/types.h>
-#include <linux/fs.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/uio.h>
-#include <linux/uaccess.h>
-
-size_t
-__filemap_copy_from_user_iovec_inatomic(char *vaddr,
-                                       const struct iovec *iov,
-                                       size_t base,
-                                       size_t bytes);
-
-/*
- * Copy as much as we can into the page and return the number of bytes which
- * were sucessfully copied.  If a fault is encountered then return the number of
- * bytes which were copied.
- */
-static inline size_t
-filemap_copy_from_user_atomic(struct page *page, unsigned long offset,
-                       const struct iovec *iov, unsigned long nr_segs,
-                       size_t base, size_t bytes)
-{
-       char *kaddr;
-       size_t copied;
-
-       kaddr = kmap_atomic(page, KM_USER0);
-       if (likely(nr_segs == 1)) {
-               int left;
-               char __user *buf = iov->iov_base + base;
-               left = __copy_from_user_inatomic_nocache(kaddr + offset,
-                                                       buf, bytes);
-               copied = bytes - left;
-       } else {
-               copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset,
-                                                       iov, base, bytes);
-       }
-       kunmap_atomic(kaddr, KM_USER0);
-
-       return copied;
-}
-
-/*
- * This has the same sideeffects and return value as
- * filemap_copy_from_user_atomic().
- * The difference is that it attempts to resolve faults.
- */
-static inline size_t
-filemap_copy_from_user(struct page *page, unsigned long offset,
-                       const struct iovec *iov, unsigned long nr_segs,
-                        size_t base, size_t bytes)
-{
-       char *kaddr;
-       size_t copied;
-
-       kaddr = kmap(page);
-       if (likely(nr_segs == 1)) {
-               int left;
-               char __user *buf = iov->iov_base + base;
-               left = __copy_from_user_nocache(kaddr + offset, buf, bytes);
-               copied = bytes - left;
-       } else {
-               copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset,
-                                                       iov, base, bytes);
-       }
-       kunmap(page);
-       return copied;
-}
-
-static inline void
-filemap_set_next_iovec(const struct iovec **iovp, unsigned long nr_segs,
-                                                size_t *basep, size_t bytes)
-{
-       if (likely(nr_segs == 1)) {
-               *basep += bytes;
-       } else {
-               const struct iovec *iov = *iovp;
-               size_t base = *basep;
-
-               while (bytes) {
-                       int copy = min(bytes, iov->iov_len - base);
-
-                       bytes -= copy;
-                       base += copy;
-                       if (iov->iov_len == base) {
-                               iov++;
-                               base = 0;
-                       }
-               }
-               *iovp = iov;
-               *basep = base;
-       }
-}
-#endif