ipc/mqueue.c: fix mq_open() return value

commit d40dcdb0172a1ba853464983a059fb45e0aaf61a upstream.

We return ENOMEM from mqueue_get_inode even when we have enough memory.
Namely in case the system rlimit of mqueue was reached.  This error
propagates to mq_queue and user sees the error unexpectedly.  So fix
this up to properly return EMFILE as described in the manpage:

	EMFILE The process already has the maximum number of files and
	       message queues open.

instead of:

	ENOMEM Insufficient memory.

With the previous patch we just switch to ERR_PTR/PTR_ERR/IS_ERR error
handling here.

Signed-off-by: Jiri Slaby <jslaby@suse.cz>
Cc: Manfred Spraul <manfred@colorfullife.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>

diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index d43c30f72f1d1c3886378be2ead31727a4c451db..ed049ea568f45ee6026941b5d8335853c33efcd8 100644 (file)
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -113,6 +113,7 @@ static struct inode *mqueue_get_inode(struct super_block *sb,
 {
        struct user_struct *u = current_user();
        struct inode *inode;
+       int ret = -ENOMEM;
 
        inode = new_inode(sb);
        if (!inode)
@@ -160,6 +161,7 @@ static struct inode *mqueue_get_inode(struct super_block *sb,
                    u->mq_bytes + mq_bytes > task_rlimit(p, RLIMIT_MSGQUEUE)) {
                        spin_unlock(&mq_lock);
                        /* mqueue_evict_inode() releases info->messages */
+                       ret = -EMFILE;
                        goto out_inode;
                }
                u->mq_bytes += mq_bytes;
@@ -179,7 +181,7 @@ static struct inode *mqueue_get_inode(struct super_block *sb,
 out_inode:
        iput(inode);
 err:
-       return NULL;
+       return ERR_PTR(ret);
 }
 
 static int mqueue_fill_super(struct super_block *sb, void *data, int silent)
@@ -195,8 +197,8 @@ static int mqueue_fill_super(struct super_block *sb, void *data, int silent)
 
        inode = mqueue_get_inode(sb, ns, S_IFDIR | S_ISVTX | S_IRWXUGO,
                                NULL);
-       if (!inode) {
-               error = -ENOMEM;
+       if (IS_ERR(inode)) {
+               error = PTR_ERR(inode);
                goto out;
        }
 
@@ -316,8 +318,8 @@ static int mqueue_create(struct inode *dir, struct dentry *dentry,
        spin_unlock(&mq_lock);
 
        inode = mqueue_get_inode(dir->i_sb, ipc_ns, mode, attr);
-       if (!inode) {
-               error = -ENOMEM;
+       if (IS_ERR(inode)) {
+               error = PTR_ERR(inode);
                spin_lock(&mq_lock);
                ipc_ns->mq_queues_count--;
                goto out_unlock;