#define MIN_TTYB_SIZE 256
#define TTYB_ALIGN_MASK 255
+/*
+ * Byte threshold to limit memory consumption for flip buffers.
+ * The actual memory limit is > 2x this amount.
+ */
+#define TTYB_MEM_LIMIT 65536
+
+/*
+ * We default to dicing tty buffer allocations to this many characters
+ * in order to avoid multiple page allocations. We know the size of
+ * tty_buffer itself but it must also be taken into account that the
+ * the buffer is 256 byte aligned. See tty_buffer_find for the allocation
+ * logic this must match
+ */
+
+#define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~0xFF)
+
+
+/**
+ * tty_buffer_lock_exclusive - gain exclusive access to buffer
+ * tty_buffer_unlock_exclusive - release exclusive access
+ *
+ * @port - tty_port owning the flip buffer
+ *
+ * Guarantees safe use of the line discipline's receive_buf() method by
+ * excluding the buffer work and any pending flush from using the flip
+ * buffer. Data can continue to be added concurrently to the flip buffer
+ * from the driver side.
+ *
+ * On release, the buffer work is restarted if there is data in the
+ * flip buffer
+ */
+
+void tty_buffer_lock_exclusive(struct tty_port *port)
+{
+ struct tty_bufhead *buf = &port->buf;
+
+ atomic_inc(&buf->priority);
+ mutex_lock(&buf->lock);
+}
+
+void tty_buffer_unlock_exclusive(struct tty_port *port)
+{
+ struct tty_bufhead *buf = &port->buf;
+ int restart;
+
+ restart = buf->head->commit != buf->head->read;
+
+ atomic_dec(&buf->priority);
+ mutex_unlock(&buf->lock);
+ if (restart)
+ queue_work(system_unbound_wq, &buf->work);
+}
+
+/**
+ * tty_buffer_space_avail - return unused buffer space
+ * @port - tty_port owning the flip buffer
+ *
+ * Returns the # of bytes which can be written by the driver without
+ * reaching the buffer limit.
+ *
+ * Note: this does not guarantee that memory is available to write
+ * the returned # of bytes (use tty_prepare_flip_string_xxx() to
+ * pre-allocate if memory guarantee is required).
+ */
+
+int tty_buffer_space_avail(struct tty_port *port)
+{
+ int space = TTYB_MEM_LIMIT - atomic_read(&port->buf.memory_used);
+ return max(space, 0);
+}
+
static void tty_buffer_reset(struct tty_buffer *p, size_t size)
{
p->used = 0;
*
* Remove all the buffers pending on a tty whether queued with data
* or in the free ring. Must be called when the tty is no longer in use
- *
- * Locking: none
*/
void tty_buffer_free_all(struct tty_port *port)
while ((p = buf->head) != NULL) {
buf->head = p->next;
- kfree(p);
+ if (p->size > 0)
+ kfree(p);
}
llist = llist_del_all(&buf->free);
llist_for_each_entry_safe(p, next, llist, free)
kfree(p);
- buf->tail = NULL;
- buf->memory_used = 0;
+ tty_buffer_reset(&buf->sentinel, 0);
+ buf->head = &buf->sentinel;
+ buf->tail = &buf->sentinel;
+
+ atomic_set(&buf->memory_used, 0);
}
/**
/* Should possibly check if this fails for the largest buffer we
have queued and recycle that ? */
- if (port->buf.memory_used + size > 65536)
+ if (atomic_read(&port->buf.memory_used) > TTYB_MEM_LIMIT)
return NULL;
p = kmalloc(sizeof(struct tty_buffer) + 2 * size, GFP_ATOMIC);
if (p == NULL)
found:
tty_buffer_reset(p, size);
- port->buf.memory_used += size;
+ atomic_add(size, &port->buf.memory_used);
return p;
}
struct tty_bufhead *buf = &port->buf;
/* Dumb strategy for now - should keep some stats */
- buf->memory_used -= b->size;
- WARN_ON(buf->memory_used < 0);
+ WARN_ON(atomic_sub_return(b->size, &buf->memory_used) < 0);
if (b->size > MIN_TTYB_SIZE)
kfree(b);
- else
+ else if (b->size > 0)
llist_add(&b->free, &buf->free);
}
/**
- * __tty_buffer_flush - flush full tty buffers
+ * tty_buffer_flush - flush full tty buffers
* @tty: tty to flush
*
- * flush all the buffers containing receive data. Caller must
- * hold the buffer lock and must have ensured no parallel flush to
- * ldisc is running.
+ * flush all the buffers containing receive data. If the buffer is
+ * being processed by flush_to_ldisc then we defer the processing
+ * to that function
*
- * Locking: Caller must hold tty->buf.lock
+ * Locking: takes buffer lock to ensure single-threaded flip buffer
+ * 'consumer'
*/
-static void __tty_buffer_flush(struct tty_port *port)
+void tty_buffer_flush(struct tty_struct *tty)
{
+ struct tty_port *port = tty->port;
struct tty_bufhead *buf = &port->buf;
struct tty_buffer *next;
- if (unlikely(buf->head == NULL))
- return;
+ atomic_inc(&buf->priority);
+
+ mutex_lock(&buf->lock);
while ((next = buf->head->next) != NULL) {
tty_buffer_free(port, buf->head);
buf->head = next;
}
- WARN_ON(buf->head != buf->tail);
buf->head->read = buf->head->commit;
-}
-
-/**
- * tty_buffer_flush - flush full tty buffers
- * @tty: tty to flush
- *
- * flush all the buffers containing receive data. If the buffer is
- * being processed by flush_to_ldisc then we defer the processing
- * to that function
- *
- * Locking: none
- */
-
-void tty_buffer_flush(struct tty_struct *tty)
-{
- struct tty_port *port = tty->port;
- struct tty_bufhead *buf = &port->buf;
- unsigned long flags;
-
- spin_lock_irqsave(&buf->lock, flags);
-
- /* If the data is being pushed to the tty layer then we can't
- process it here. Instead set a flag and the flush_to_ldisc
- path will process the flush request before it exits */
- if (test_bit(TTYP_FLUSHING, &port->iflags)) {
- set_bit(TTYP_FLUSHPENDING, &port->iflags);
- spin_unlock_irqrestore(&buf->lock, flags);
- wait_event(tty->read_wait,
- test_bit(TTYP_FLUSHPENDING, &port->iflags) == 0);
- return;
- } else
- __tty_buffer_flush(port);
- spin_unlock_irqrestore(&buf->lock, flags);
+ atomic_dec(&buf->priority);
+ mutex_unlock(&buf->lock);
}
/**
*
* Make at least size bytes of linear space available for the tty
* buffer. If we fail return the size we managed to find.
- *
- * Locking: Takes port->buf.lock
*/
int tty_buffer_request_room(struct tty_port *port, size_t size)
{
struct tty_bufhead *buf = &port->buf;
struct tty_buffer *b, *n;
int left;
- unsigned long flags;
- spin_lock_irqsave(&buf->lock, flags);
- /* OPTIMISATION: We could keep a per tty "zero" sized buffer to
- remove this conditional if its worth it. This would be invisible
- to the callers */
+
b = buf->tail;
- if (b != NULL)
- left = b->size - b->used;
- else
- left = 0;
+ left = b->size - b->used;
if (left < size) {
/* This is the slow path - looking for new buffers to use */
if ((n = tty_buffer_alloc(port, size)) != NULL) {
- if (b != NULL) {
- b->next = n;
- b->commit = b->used;
- } else
- buf->head = n;
buf->tail = n;
+ b->commit = b->used;
+ smp_mb();
+ b->next = n;
} else
size = left;
}
- spin_unlock_irqrestore(&buf->lock, flags);
return size;
}
EXPORT_SYMBOL_GPL(tty_buffer_request_room);
*
* Queue a series of bytes to the tty buffering. All the characters
* passed are marked with the supplied flag. Returns the number added.
- *
- * Locking: Called functions may take port->buf.lock
*/
int tty_insert_flip_string_fixed_flag(struct tty_port *port,
int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
int space = tty_buffer_request_room(port, goal);
struct tty_buffer *tb = port->buf.tail;
- /* If there is no space then tb may be NULL */
- if (unlikely(space == 0)) {
+ if (unlikely(space == 0))
break;
- }
memcpy(char_buf_ptr(tb, tb->used), chars, space);
memset(flag_buf_ptr(tb, tb->used), flag, space);
tb->used += space;
* Queue a series of bytes to the tty buffering. For each character
* the flags array indicates the status of the character. Returns the
* number added.
- *
- * Locking: Called functions may take port->buf.lock
*/
int tty_insert_flip_string_flags(struct tty_port *port,
int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
int space = tty_buffer_request_room(port, goal);
struct tty_buffer *tb = port->buf.tail;
- /* If there is no space then tb may be NULL */
- if (unlikely(space == 0)) {
+ if (unlikely(space == 0))
break;
- }
memcpy(char_buf_ptr(tb, tb->used), chars, space);
memcpy(flag_buf_ptr(tb, tb->used), flags, space);
tb->used += space;
* processing by the line discipline.
* Note that this function can only be used when the low_latency flag
* is unset. Otherwise the workqueue won't be flushed.
- *
- * Locking: Takes port->buf.lock
*/
void tty_schedule_flip(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
- unsigned long flags;
WARN_ON(port->low_latency);
- spin_lock_irqsave(&buf->lock, flags);
- if (buf->tail != NULL)
- buf->tail->commit = buf->tail->used;
- spin_unlock_irqrestore(&buf->lock, flags);
+ buf->tail->commit = buf->tail->used;
schedule_work(&buf->work);
}
EXPORT_SYMBOL(tty_schedule_flip);
* accounted for as ready for normal characters. This is used for drivers
* that need their own block copy routines into the buffer. There is no
* guarantee the buffer is a DMA target!
- *
- * Locking: May call functions taking port->buf.lock
*/
int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars,
* accounted for as ready for characters. This is used for drivers
* that need their own block copy routines into the buffer. There is no
* guarantee the buffer is a DMA target!
- *
- * Locking: May call functions taking port->buf.lock
*/
int tty_prepare_flip_string_flags(struct tty_port *port,
* This routine is called out of the software interrupt to flush data
* from the buffer chain to the line discipline.
*
- * Locking: holds tty->buf.lock to guard buffer list. Drops the lock
- * while invoking the line discipline receive_buf method. The
- * receive_buf method is single threaded for each tty instance.
+ * The receive_buf method is single threaded for each tty instance.
+ *
+ * Locking: takes buffer lock to ensure single-threaded flip buffer
+ * 'consumer'
*/
static void flush_to_ldisc(struct work_struct *work)
struct tty_port *port = container_of(work, struct tty_port, buf.work);
struct tty_bufhead *buf = &port->buf;
struct tty_struct *tty;
- unsigned long flags;
struct tty_ldisc *disc;
tty = port->itty;
if (disc == NULL)
return;
- spin_lock_irqsave(&buf->lock, flags);
-
- if (!test_and_set_bit(TTYP_FLUSHING, &port->iflags)) {
- struct tty_buffer *head;
- while ((head = buf->head) != NULL) {
- int count;
-
- count = head->commit - head->read;
- if (!count) {
- if (head->next == NULL)
- break;
- buf->head = head->next;
- tty_buffer_free(port, head);
- continue;
- }
- spin_unlock_irqrestore(&buf->lock, flags);
-
- count = receive_buf(tty, head, count);
-
- spin_lock_irqsave(&buf->lock, flags);
- /* Ldisc or user is trying to flush the buffers.
- We may have a deferred request to flush the
- input buffer, if so pull the chain under the lock
- and empty the queue */
- if (test_bit(TTYP_FLUSHPENDING, &port->iflags)) {
- __tty_buffer_flush(port);
- clear_bit(TTYP_FLUSHPENDING, &port->iflags);
- wake_up(&tty->read_wait);
- break;
- } else if (!count)
+ mutex_lock(&buf->lock);
+
+ while (1) {
+ struct tty_buffer *head = buf->head;
+ int count;
+
+ /* Ldisc or user is trying to gain exclusive access */
+ if (atomic_read(&buf->priority))
+ break;
+
+ count = head->commit - head->read;
+ if (!count) {
+ if (head->next == NULL)
break;
+ buf->head = head->next;
+ tty_buffer_free(port, head);
+ continue;
}
- clear_bit(TTYP_FLUSHING, &port->iflags);
+
+ count = receive_buf(tty, head, count);
+ if (!count)
+ break;
}
- spin_unlock_irqrestore(&buf->lock, flags);
+ mutex_unlock(&buf->lock);
tty_ldisc_deref(disc);
}
*
* In the event of the queue being busy for flipping the work will be
* held off and retried later.
- *
- * Locking: tty buffer lock. Driver locks in low latency mode.
*/
void tty_flip_buffer_push(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
- unsigned long flags;
- spin_lock_irqsave(&buf->lock, flags);
- if (buf->tail != NULL)
- buf->tail->commit = buf->tail->used;
- spin_unlock_irqrestore(&buf->lock, flags);
+ buf->tail->commit = buf->tail->used;
if (port->low_latency)
flush_to_ldisc(&buf->work);
*
* Set up the initial state of the buffer management for a tty device.
* Must be called before the other tty buffer functions are used.
- *
- * Locking: none
*/
void tty_buffer_init(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
- spin_lock_init(&buf->lock);
- buf->head = NULL;
- buf->tail = NULL;
+ mutex_init(&buf->lock);
+ tty_buffer_reset(&buf->sentinel, 0);
+ buf->head = &buf->sentinel;
+ buf->tail = &buf->sentinel;
init_llist_head(&buf->free);
- buf->memory_used = 0;
+ atomic_set(&buf->memory_used, 0);
+ atomic_set(&buf->priority, 0);
INIT_WORK(&buf->work, flush_to_ldisc);
}
-
projects / firefly-linux-kernel-4.4.55.git / blobdiff