#include <linux/module.h>
#include <linux/ratelimit.h>
+
+#define MIN_TTYB_SIZE 256
+#define TTYB_ALIGN_MASK 255
+
+/*
+ * Byte threshold to limit memory consumption for flip buffers.
+ * The actual memory limit is > 2x this amount.
+ */
+#define TTYB_MEM_LIMIT 65536
+
+/*
+ * We default to dicing tty buffer allocations to this many characters
+ * in order to avoid multiple page allocations. We know the size of
+ * tty_buffer itself but it must also be taken into account that the
+ * the buffer is 256 byte aligned. See tty_buffer_find for the allocation
+ * logic this must match
+ */
+
+#define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~0xFF)
+
+
+/**
+ * tty_buffer_lock_exclusive - gain exclusive access to buffer
+ * tty_buffer_unlock_exclusive - release exclusive access
+ *
+ * @port - tty_port owning the flip buffer
+ *
+ * Guarantees safe use of the line discipline's receive_buf() method by
+ * excluding the buffer work and any pending flush from using the flip
+ * buffer. Data can continue to be added concurrently to the flip buffer
+ * from the driver side.
+ *
+ * On release, the buffer work is restarted if there is data in the
+ * flip buffer
+ */
+
+void tty_buffer_lock_exclusive(struct tty_port *port)
+{
+ struct tty_bufhead *buf = &port->buf;
+
+ atomic_inc(&buf->priority);
+ mutex_lock(&buf->lock);
+}
+
+void tty_buffer_unlock_exclusive(struct tty_port *port)
+{
+ struct tty_bufhead *buf = &port->buf;
+ int restart;
+
+ restart = buf->head->commit != buf->head->read;
+
+ atomic_dec(&buf->priority);
+ mutex_unlock(&buf->lock);
+ if (restart)
+ queue_work(system_unbound_wq, &buf->work);
+}
+
+/**
+ * tty_buffer_space_avail - return unused buffer space
+ * @port - tty_port owning the flip buffer
+ *
+ * Returns the # of bytes which can be written by the driver without
+ * reaching the buffer limit.
+ *
+ * Note: this does not guarantee that memory is available to write
+ * the returned # of bytes (use tty_prepare_flip_string_xxx() to
+ * pre-allocate if memory guarantee is required).
+ */
+
+int tty_buffer_space_avail(struct tty_port *port)
+{
+ int space = TTYB_MEM_LIMIT - atomic_read(&port->buf.memory_used);
+ return max(space, 0);
+}
+
+static void tty_buffer_reset(struct tty_buffer *p, size_t size)
+{
+ p->used = 0;
+ p->size = size;
+ p->next = NULL;
+ p->commit = 0;
+ p->read = 0;
+}
+
/**
* tty_buffer_free_all - free buffers used by a tty
* @tty: tty to free from
*
* Remove all the buffers pending on a tty whether queued with data
* or in the free ring. Must be called when the tty is no longer in use
- *
- * Locking: none
*/
void tty_buffer_free_all(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
- struct tty_buffer *thead;
+ struct tty_buffer *p, *next;
+ struct llist_node *llist;
- while ((thead = buf->head) != NULL) {
- buf->head = thead->next;
- kfree(thead);
- }
- while ((thead = buf->free) != NULL) {
- buf->free = thead->next;
- kfree(thead);
+ while ((p = buf->head) != NULL) {
+ buf->head = p->next;
+ if (p->size > 0)
+ kfree(p);
}
- buf->tail = NULL;
- buf->memory_used = 0;
+ llist = llist_del_all(&buf->free);
+ llist_for_each_entry_safe(p, next, llist, free)
+ kfree(p);
+
+ tty_buffer_reset(&buf->sentinel, 0);
+ buf->head = &buf->sentinel;
+ buf->tail = &buf->sentinel;
+
+ atomic_set(&buf->memory_used, 0);
}
/**
* @size: desired size (characters)
*
* Allocate a new tty buffer to hold the desired number of characters.
+ * We round our buffers off in 256 character chunks to get better
+ * allocation behaviour.
* Return NULL if out of memory or the allocation would exceed the
* per device queue
- *
- * Locking: Caller must hold tty->buf.lock
*/
static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size)
{
+ struct llist_node *free;
struct tty_buffer *p;
- if (port->buf.memory_used + size > 65536)
+ /* Round the buffer size out */
+ size = __ALIGN_MASK(size, TTYB_ALIGN_MASK);
+
+ if (size <= MIN_TTYB_SIZE) {
+ free = llist_del_first(&port->buf.free);
+ if (free) {
+ p = llist_entry(free, struct tty_buffer, free);
+ goto found;
+ }
+ }
+
+ /* Should possibly check if this fails for the largest buffer we
+ have queued and recycle that ? */
+ if (atomic_read(&port->buf.memory_used) > TTYB_MEM_LIMIT)
return NULL;
p = kmalloc(sizeof(struct tty_buffer) + 2 * size, GFP_ATOMIC);
if (p == NULL)
return NULL;
- p->used = 0;
- p->size = size;
- p->next = NULL;
- p->commit = 0;
- p->read = 0;
- p->char_buf_ptr = (char *)(p->data);
- p->flag_buf_ptr = (unsigned char *)p->char_buf_ptr + size;
- port->buf.memory_used += size;
+
+found:
+ tty_buffer_reset(p, size);
+ atomic_add(size, &port->buf.memory_used);
return p;
}
*
* Free a tty buffer, or add it to the free list according to our
* internal strategy
- *
- * Locking: Caller must hold tty->buf.lock
*/
static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b)
struct tty_bufhead *buf = &port->buf;
/* Dumb strategy for now - should keep some stats */
- buf->memory_used -= b->size;
- WARN_ON(buf->memory_used < 0);
+ WARN_ON(atomic_sub_return(b->size, &buf->memory_used) < 0);
- if (b->size >= 512)
+ if (b->size > MIN_TTYB_SIZE)
kfree(b);
- else {
- b->next = buf->free;
- buf->free = b;
- }
-}
-
-/**
- * __tty_buffer_flush - flush full tty buffers
- * @tty: tty to flush
- *
- * flush all the buffers containing receive data. Caller must
- * hold the buffer lock and must have ensured no parallel flush to
- * ldisc is running.
- *
- * Locking: Caller must hold tty->buf.lock
- */
-
-static void __tty_buffer_flush(struct tty_port *port)
-{
- struct tty_bufhead *buf = &port->buf;
- struct tty_buffer *thead;
-
- if (unlikely(buf->head == NULL))
- return;
- while ((thead = buf->head->next) != NULL) {
- tty_buffer_free(port, buf->head);
- buf->head = thead;
- }
- WARN_ON(buf->head != buf->tail);
- buf->head->read = buf->head->commit;
+ else if (b->size > 0)
+ llist_add(&b->free, &buf->free);
}
/**
* being processed by flush_to_ldisc then we defer the processing
* to that function
*
- * Locking: none
+ * Locking: takes buffer lock to ensure single-threaded flip buffer
+ * 'consumer'
*/
void tty_buffer_flush(struct tty_struct *tty)
{
struct tty_port *port = tty->port;
struct tty_bufhead *buf = &port->buf;
- unsigned long flags;
-
- spin_lock_irqsave(&buf->lock, flags);
-
- /* If the data is being pushed to the tty layer then we can't
- process it here. Instead set a flag and the flush_to_ldisc
- path will process the flush request before it exits */
- if (test_bit(TTYP_FLUSHING, &port->iflags)) {
- set_bit(TTYP_FLUSHPENDING, &port->iflags);
- spin_unlock_irqrestore(&buf->lock, flags);
- wait_event(tty->read_wait,
- test_bit(TTYP_FLUSHPENDING, &port->iflags) == 0);
- return;
- } else
- __tty_buffer_flush(port);
- spin_unlock_irqrestore(&buf->lock, flags);
-}
+ struct tty_buffer *next;
-/**
- * tty_buffer_find - find a free tty buffer
- * @tty: tty owning the buffer
- * @size: characters wanted
- *
- * Locate an existing suitable tty buffer or if we are lacking one then
- * allocate a new one. We round our buffers off in 256 character chunks
- * to get better allocation behaviour.
- *
- * Locking: Caller must hold tty->buf.lock
- */
+ atomic_inc(&buf->priority);
-static struct tty_buffer *tty_buffer_find(struct tty_port *port, size_t size)
-{
- struct tty_buffer **tbh = &port->buf.free;
- while ((*tbh) != NULL) {
- struct tty_buffer *t = *tbh;
- if (t->size >= size) {
- *tbh = t->next;
- t->next = NULL;
- t->used = 0;
- t->commit = 0;
- t->read = 0;
- port->buf.memory_used += t->size;
- return t;
- }
- tbh = &((*tbh)->next);
+ mutex_lock(&buf->lock);
+ while ((next = buf->head->next) != NULL) {
+ tty_buffer_free(port, buf->head);
+ buf->head = next;
}
- /* Round the buffer size out */
- size = (size + 0xFF) & ~0xFF;
- return tty_buffer_alloc(port, size);
- /* Should possibly check if this fails for the largest buffer we
- have queued and recycle that ? */
+ buf->head->read = buf->head->commit;
+ atomic_dec(&buf->priority);
+ mutex_unlock(&buf->lock);
}
+
/**
* tty_buffer_request_room - grow tty buffer if needed
* @tty: tty structure
*
* Make at least size bytes of linear space available for the tty
* buffer. If we fail return the size we managed to find.
- *
- * Locking: Takes port->buf.lock
*/
int tty_buffer_request_room(struct tty_port *port, size_t size)
{
struct tty_bufhead *buf = &port->buf;
struct tty_buffer *b, *n;
int left;
- unsigned long flags;
- spin_lock_irqsave(&buf->lock, flags);
- /* OPTIMISATION: We could keep a per tty "zero" sized buffer to
- remove this conditional if its worth it. This would be invisible
- to the callers */
+
b = buf->tail;
- if (b != NULL)
- left = b->size - b->used;
- else
- left = 0;
+ left = b->size - b->used;
if (left < size) {
/* This is the slow path - looking for new buffers to use */
- if ((n = tty_buffer_find(port, size)) != NULL) {
- if (b != NULL) {
- b->next = n;
- b->commit = b->used;
- } else
- buf->head = n;
+ if ((n = tty_buffer_alloc(port, size)) != NULL) {
buf->tail = n;
+ b->commit = b->used;
+ smp_mb();
+ b->next = n;
} else
size = left;
}
- spin_unlock_irqrestore(&buf->lock, flags);
return size;
}
EXPORT_SYMBOL_GPL(tty_buffer_request_room);
*
* Queue a series of bytes to the tty buffering. All the characters
* passed are marked with the supplied flag. Returns the number added.
- *
- * Locking: Called functions may take port->buf.lock
*/
int tty_insert_flip_string_fixed_flag(struct tty_port *port,
int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
int space = tty_buffer_request_room(port, goal);
struct tty_buffer *tb = port->buf.tail;
- /* If there is no space then tb may be NULL */
- if (unlikely(space == 0)) {
+ if (unlikely(space == 0))
break;
- }
- memcpy(tb->char_buf_ptr + tb->used, chars, space);
- memset(tb->flag_buf_ptr + tb->used, flag, space);
+ memcpy(char_buf_ptr(tb, tb->used), chars, space);
+ memset(flag_buf_ptr(tb, tb->used), flag, space);
tb->used += space;
copied += space;
chars += space;
* Queue a series of bytes to the tty buffering. For each character
* the flags array indicates the status of the character. Returns the
* number added.
- *
- * Locking: Called functions may take port->buf.lock
*/
int tty_insert_flip_string_flags(struct tty_port *port,
int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
int space = tty_buffer_request_room(port, goal);
struct tty_buffer *tb = port->buf.tail;
- /* If there is no space then tb may be NULL */
- if (unlikely(space == 0)) {
+ if (unlikely(space == 0))
break;
- }
- memcpy(tb->char_buf_ptr + tb->used, chars, space);
- memcpy(tb->flag_buf_ptr + tb->used, flags, space);
+ memcpy(char_buf_ptr(tb, tb->used), chars, space);
+ memcpy(flag_buf_ptr(tb, tb->used), flags, space);
tb->used += space;
copied += space;
chars += space;
* processing by the line discipline.
* Note that this function can only be used when the low_latency flag
* is unset. Otherwise the workqueue won't be flushed.
- *
- * Locking: Takes port->buf.lock
*/
void tty_schedule_flip(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
- unsigned long flags;
WARN_ON(port->low_latency);
- spin_lock_irqsave(&buf->lock, flags);
- if (buf->tail != NULL)
- buf->tail->commit = buf->tail->used;
- spin_unlock_irqrestore(&buf->lock, flags);
+ buf->tail->commit = buf->tail->used;
schedule_work(&buf->work);
}
EXPORT_SYMBOL(tty_schedule_flip);
* accounted for as ready for normal characters. This is used for drivers
* that need their own block copy routines into the buffer. There is no
* guarantee the buffer is a DMA target!
- *
- * Locking: May call functions taking port->buf.lock
*/
int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars,
int space = tty_buffer_request_room(port, size);
if (likely(space)) {
struct tty_buffer *tb = port->buf.tail;
- *chars = tb->char_buf_ptr + tb->used;
- memset(tb->flag_buf_ptr + tb->used, TTY_NORMAL, space);
+ *chars = char_buf_ptr(tb, tb->used);
+ memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space);
tb->used += space;
}
return space;
* accounted for as ready for characters. This is used for drivers
* that need their own block copy routines into the buffer. There is no
* guarantee the buffer is a DMA target!
- *
- * Locking: May call functions taking port->buf.lock
*/
int tty_prepare_flip_string_flags(struct tty_port *port,
int space = tty_buffer_request_room(port, size);
if (likely(space)) {
struct tty_buffer *tb = port->buf.tail;
- *chars = tb->char_buf_ptr + tb->used;
- *flags = tb->flag_buf_ptr + tb->used;
+ *chars = char_buf_ptr(tb, tb->used);
+ *flags = flag_buf_ptr(tb, tb->used);
tb->used += space;
}
return space;
EXPORT_SYMBOL_GPL(tty_prepare_flip_string_flags);
+static int
+receive_buf(struct tty_struct *tty, struct tty_buffer *head, int count)
+{
+ struct tty_ldisc *disc = tty->ldisc;
+ unsigned char *p = char_buf_ptr(head, head->read);
+ char *f = flag_buf_ptr(head, head->read);
+
+ if (disc->ops->receive_buf2)
+ count = disc->ops->receive_buf2(tty, p, f, count);
+ else {
+ count = min_t(int, count, tty->receive_room);
+ if (count)
+ disc->ops->receive_buf(tty, p, f, count);
+ }
+ head->read += count;
+ return count;
+}
/**
* flush_to_ldisc
* This routine is called out of the software interrupt to flush data
* from the buffer chain to the line discipline.
*
- * Locking: holds tty->buf.lock to guard buffer list. Drops the lock
- * while invoking the line discipline receive_buf method. The
- * receive_buf method is single threaded for each tty instance.
+ * The receive_buf method is single threaded for each tty instance.
+ *
+ * Locking: takes buffer lock to ensure single-threaded flip buffer
+ * 'consumer'
*/
static void flush_to_ldisc(struct work_struct *work)
struct tty_port *port = container_of(work, struct tty_port, buf.work);
struct tty_bufhead *buf = &port->buf;
struct tty_struct *tty;
- unsigned long flags;
struct tty_ldisc *disc;
tty = port->itty;
return;
disc = tty_ldisc_ref(tty);
- if (disc == NULL) /* !TTY_LDISC */
+ if (disc == NULL)
return;
- spin_lock_irqsave(&buf->lock, flags);
-
- if (!test_and_set_bit(TTYP_FLUSHING, &port->iflags)) {
- struct tty_buffer *head;
- while ((head = buf->head) != NULL) {
- int count;
- char *char_buf;
- unsigned char *flag_buf;
-
- count = head->commit - head->read;
- if (!count) {
- if (head->next == NULL)
- break;
- buf->head = head->next;
- tty_buffer_free(port, head);
- continue;
- }
- if (!tty->receive_room)
- break;
- if (count > tty->receive_room)
- count = tty->receive_room;
- char_buf = head->char_buf_ptr + head->read;
- flag_buf = head->flag_buf_ptr + head->read;
- head->read += count;
- spin_unlock_irqrestore(&buf->lock, flags);
- disc->ops->receive_buf(tty, char_buf,
- flag_buf, count);
- spin_lock_irqsave(&buf->lock, flags);
- /* Ldisc or user is trying to flush the buffers.
- We may have a deferred request to flush the
- input buffer, if so pull the chain under the lock
- and empty the queue */
- if (test_bit(TTYP_FLUSHPENDING, &port->iflags)) {
- __tty_buffer_flush(port);
- clear_bit(TTYP_FLUSHPENDING, &port->iflags);
- wake_up(&tty->read_wait);
+ mutex_lock(&buf->lock);
+
+ while (1) {
+ struct tty_buffer *head = buf->head;
+ int count;
+
+ /* Ldisc or user is trying to gain exclusive access */
+ if (atomic_read(&buf->priority))
+ break;
+
+ count = head->commit - head->read;
+ if (!count) {
+ if (head->next == NULL)
break;
- }
+ buf->head = head->next;
+ tty_buffer_free(port, head);
+ continue;
}
- clear_bit(TTYP_FLUSHING, &port->iflags);
+
+ count = receive_buf(tty, head, count);
+ if (!count)
+ break;
}
- spin_unlock_irqrestore(&buf->lock, flags);
+ mutex_unlock(&buf->lock);
tty_ldisc_deref(disc);
}
*
* In the event of the queue being busy for flipping the work will be
* held off and retried later.
- *
- * Locking: tty buffer lock. Driver locks in low latency mode.
*/
void tty_flip_buffer_push(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
- unsigned long flags;
- spin_lock_irqsave(&buf->lock, flags);
- if (buf->tail != NULL)
- buf->tail->commit = buf->tail->used;
- spin_unlock_irqrestore(&buf->lock, flags);
+ buf->tail->commit = buf->tail->used;
if (port->low_latency)
flush_to_ldisc(&buf->work);
*
* Set up the initial state of the buffer management for a tty device.
* Must be called before the other tty buffer functions are used.
- *
- * Locking: none
*/
void tty_buffer_init(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
- spin_lock_init(&buf->lock);
- buf->head = NULL;
- buf->tail = NULL;
- buf->free = NULL;
- buf->memory_used = 0;
+ mutex_init(&buf->lock);
+ tty_buffer_reset(&buf->sentinel, 0);
+ buf->head = &buf->sentinel;
+ buf->tail = &buf->sentinel;
+ init_llist_head(&buf->free);
+ atomic_set(&buf->memory_used, 0);
+ atomic_set(&buf->priority, 0);
INIT_WORK(&buf->work, flush_to_ldisc);
}
-
projects / firefly-linux-kernel-4.4.55.git / blobdiff