2 * Tty buffer allocation management
5 #include <linux/types.h>
6 #include <linux/errno.h>
8 #include <linux/tty_driver.h>
9 #include <linux/tty_flip.h>
10 #include <linux/timer.h>
11 #include <linux/string.h>
12 #include <linux/slab.h>
13 #include <linux/sched.h>
14 #include <linux/init.h>
15 #include <linux/wait.h>
16 #include <linux/bitops.h>
17 #include <linux/delay.h>
18 #include <linux/module.h>
19 #include <linux/ratelimit.h>
22 #define MIN_TTYB_SIZE 256
23 #define TTYB_ALIGN_MASK 255
26 * Byte threshold to limit memory consumption for flip buffers.
27 * The actual memory limit is > 2x this amount.
29 #define TTYB_MEM_LIMIT 65536
33 * tty_buffer_lock_exclusive - gain exclusive access to buffer
34 * tty_buffer_unlock_exclusive - release exclusive access
36 * @port - tty_port owning the flip buffer
38 * Guarantees safe use of the line discipline's receive_buf() method by
39 * excluding the buffer work and any pending flush from using the flip
40 * buffer. Data can continue to be added concurrently to the flip buffer
41 * from the driver side.
43 * On release, the buffer work is restarted if there is data in the
47 void tty_buffer_lock_exclusive(struct tty_port *port)
49 struct tty_bufhead *buf = &port->buf;
51 atomic_inc(&buf->priority);
52 mutex_lock(&buf->lock);
55 void tty_buffer_unlock_exclusive(struct tty_port *port)
57 struct tty_bufhead *buf = &port->buf;
60 restart = buf->head->commit != buf->head->read;
62 atomic_dec(&buf->priority);
63 mutex_unlock(&buf->lock);
65 queue_work(system_unbound_wq, &buf->work);
69 * tty_buffer_space_avail - return unused buffer space
70 * @port - tty_port owning the flip buffer
72 * Returns the # of bytes which can be written by the driver without
73 * reaching the buffer limit.
75 * Note: this does not guarantee that memory is available to write
76 * the returned # of bytes (use tty_prepare_flip_string_xxx() to
77 * pre-allocate if memory guarantee is required).
80 int tty_buffer_space_avail(struct tty_port *port)
82 int space = TTYB_MEM_LIMIT - atomic_read(&port->buf.memory_used);
86 static void tty_buffer_reset(struct tty_buffer *p, size_t size)
96 * tty_buffer_free_all - free buffers used by a tty
97 * @tty: tty to free from
99 * Remove all the buffers pending on a tty whether queued with data
100 * or in the free ring. Must be called when the tty is no longer in use
103 void tty_buffer_free_all(struct tty_port *port)
105 struct tty_bufhead *buf = &port->buf;
106 struct tty_buffer *p, *next;
107 struct llist_node *llist;
109 while ((p = buf->head) != NULL) {
114 llist = llist_del_all(&buf->free);
115 llist_for_each_entry_safe(p, next, llist, free)
118 tty_buffer_reset(&buf->sentinel, 0);
119 buf->head = &buf->sentinel;
120 buf->tail = &buf->sentinel;
122 atomic_set(&buf->memory_used, 0);
126 * tty_buffer_alloc - allocate a tty buffer
128 * @size: desired size (characters)
130 * Allocate a new tty buffer to hold the desired number of characters.
131 * We round our buffers off in 256 character chunks to get better
132 * allocation behaviour.
133 * Return NULL if out of memory or the allocation would exceed the
137 static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size)
139 struct llist_node *free;
140 struct tty_buffer *p;
142 /* Round the buffer size out */
143 size = __ALIGN_MASK(size, TTYB_ALIGN_MASK);
145 if (size <= MIN_TTYB_SIZE) {
146 free = llist_del_first(&port->buf.free);
148 p = llist_entry(free, struct tty_buffer, free);
153 /* Should possibly check if this fails for the largest buffer we
154 have queued and recycle that ? */
155 if (atomic_read(&port->buf.memory_used) > TTYB_MEM_LIMIT)
157 p = kmalloc(sizeof(struct tty_buffer) + 2 * size, GFP_ATOMIC);
162 tty_buffer_reset(p, size);
163 atomic_add(size, &port->buf.memory_used);
168 * tty_buffer_free - free a tty buffer
169 * @tty: tty owning the buffer
170 * @b: the buffer to free
172 * Free a tty buffer, or add it to the free list according to our
176 static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b)
178 struct tty_bufhead *buf = &port->buf;
180 /* Dumb strategy for now - should keep some stats */
181 WARN_ON(atomic_sub_return(b->size, &buf->memory_used) < 0);
183 if (b->size > MIN_TTYB_SIZE)
185 else if (b->size > 0)
186 llist_add(&b->free, &buf->free);
190 * tty_buffer_flush - flush full tty buffers
193 * flush all the buffers containing receive data. If the buffer is
194 * being processed by flush_to_ldisc then we defer the processing
197 * Locking: takes buffer lock to ensure single-threaded flip buffer
201 void tty_buffer_flush(struct tty_struct *tty)
203 struct tty_port *port = tty->port;
204 struct tty_bufhead *buf = &port->buf;
205 struct tty_buffer *next;
207 atomic_inc(&buf->priority);
209 mutex_lock(&buf->lock);
210 while ((next = buf->head->next) != NULL) {
211 tty_buffer_free(port, buf->head);
214 buf->head->read = buf->head->commit;
215 atomic_dec(&buf->priority);
216 mutex_unlock(&buf->lock);
220 * tty_buffer_request_room - grow tty buffer if needed
221 * @tty: tty structure
222 * @size: size desired
224 * Make at least size bytes of linear space available for the tty
225 * buffer. If we fail return the size we managed to find.
227 int tty_buffer_request_room(struct tty_port *port, size_t size)
229 struct tty_bufhead *buf = &port->buf;
230 struct tty_buffer *b, *n;
234 left = b->size - b->used;
237 /* This is the slow path - looking for new buffers to use */
238 if ((n = tty_buffer_alloc(port, size)) != NULL) {
248 EXPORT_SYMBOL_GPL(tty_buffer_request_room);
251 * tty_insert_flip_string_fixed_flag - Add characters to the tty buffer
254 * @flag: flag value for each character
257 * Queue a series of bytes to the tty buffering. All the characters
258 * passed are marked with the supplied flag. Returns the number added.
261 int tty_insert_flip_string_fixed_flag(struct tty_port *port,
262 const unsigned char *chars, char flag, size_t size)
266 int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
267 int space = tty_buffer_request_room(port, goal);
268 struct tty_buffer *tb = port->buf.tail;
269 if (unlikely(space == 0))
271 memcpy(char_buf_ptr(tb, tb->used), chars, space);
272 memset(flag_buf_ptr(tb, tb->used), flag, space);
276 /* There is a small chance that we need to split the data over
277 several buffers. If this is the case we must loop */
278 } while (unlikely(size > copied));
281 EXPORT_SYMBOL(tty_insert_flip_string_fixed_flag);
284 * tty_insert_flip_string_flags - Add characters to the tty buffer
290 * Queue a series of bytes to the tty buffering. For each character
291 * the flags array indicates the status of the character. Returns the
295 int tty_insert_flip_string_flags(struct tty_port *port,
296 const unsigned char *chars, const char *flags, size_t size)
300 int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
301 int space = tty_buffer_request_room(port, goal);
302 struct tty_buffer *tb = port->buf.tail;
303 if (unlikely(space == 0))
305 memcpy(char_buf_ptr(tb, tb->used), chars, space);
306 memcpy(flag_buf_ptr(tb, tb->used), flags, space);
311 /* There is a small chance that we need to split the data over
312 several buffers. If this is the case we must loop */
313 } while (unlikely(size > copied));
316 EXPORT_SYMBOL(tty_insert_flip_string_flags);
319 * tty_schedule_flip - push characters to ldisc
320 * @port: tty port to push from
322 * Takes any pending buffers and transfers their ownership to the
323 * ldisc side of the queue. It then schedules those characters for
324 * processing by the line discipline.
325 * Note that this function can only be used when the low_latency flag
326 * is unset. Otherwise the workqueue won't be flushed.
329 void tty_schedule_flip(struct tty_port *port)
331 struct tty_bufhead *buf = &port->buf;
332 WARN_ON(port->low_latency);
334 buf->tail->commit = buf->tail->used;
335 schedule_work(&buf->work);
337 EXPORT_SYMBOL(tty_schedule_flip);
340 * tty_prepare_flip_string - make room for characters
342 * @chars: return pointer for character write area
343 * @size: desired size
345 * Prepare a block of space in the buffer for data. Returns the length
346 * available and buffer pointer to the space which is now allocated and
347 * accounted for as ready for normal characters. This is used for drivers
348 * that need their own block copy routines into the buffer. There is no
349 * guarantee the buffer is a DMA target!
352 int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars,
355 int space = tty_buffer_request_room(port, size);
357 struct tty_buffer *tb = port->buf.tail;
358 *chars = char_buf_ptr(tb, tb->used);
359 memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space);
364 EXPORT_SYMBOL_GPL(tty_prepare_flip_string);
367 * tty_prepare_flip_string_flags - make room for characters
369 * @chars: return pointer for character write area
370 * @flags: return pointer for status flag write area
371 * @size: desired size
373 * Prepare a block of space in the buffer for data. Returns the length
374 * available and buffer pointer to the space which is now allocated and
375 * accounted for as ready for characters. This is used for drivers
376 * that need their own block copy routines into the buffer. There is no
377 * guarantee the buffer is a DMA target!
380 int tty_prepare_flip_string_flags(struct tty_port *port,
381 unsigned char **chars, char **flags, size_t size)
383 int space = tty_buffer_request_room(port, size);
385 struct tty_buffer *tb = port->buf.tail;
386 *chars = char_buf_ptr(tb, tb->used);
387 *flags = flag_buf_ptr(tb, tb->used);
392 EXPORT_SYMBOL_GPL(tty_prepare_flip_string_flags);
396 receive_buf(struct tty_struct *tty, struct tty_buffer *head, int count)
398 struct tty_ldisc *disc = tty->ldisc;
399 unsigned char *p = char_buf_ptr(head, head->read);
400 char *f = flag_buf_ptr(head, head->read);
402 if (disc->ops->receive_buf2)
403 count = disc->ops->receive_buf2(tty, p, f, count);
405 count = min_t(int, count, tty->receive_room);
407 disc->ops->receive_buf(tty, p, f, count);
415 * @work: tty structure passed from work queue.
417 * This routine is called out of the software interrupt to flush data
418 * from the buffer chain to the line discipline.
420 * The receive_buf method is single threaded for each tty instance.
422 * Locking: takes buffer lock to ensure single-threaded flip buffer
426 static void flush_to_ldisc(struct work_struct *work)
428 struct tty_port *port = container_of(work, struct tty_port, buf.work);
429 struct tty_bufhead *buf = &port->buf;
430 struct tty_struct *tty;
431 struct tty_ldisc *disc;
437 disc = tty_ldisc_ref(tty);
441 mutex_lock(&buf->lock);
444 struct tty_buffer *head = buf->head;
447 /* Ldisc or user is trying to gain exclusive access */
448 if (atomic_read(&buf->priority))
451 count = head->commit - head->read;
453 if (head->next == NULL)
455 buf->head = head->next;
456 tty_buffer_free(port, head);
460 count = receive_buf(tty, head, count);
465 mutex_unlock(&buf->lock);
467 tty_ldisc_deref(disc);
474 * Push the terminal flip buffers to the line discipline.
476 * Must not be called from IRQ context.
478 void tty_flush_to_ldisc(struct tty_struct *tty)
480 if (!tty->port->low_latency)
481 flush_work(&tty->port->buf.work);
485 * tty_flip_buffer_push - terminal
486 * @port: tty port to push
488 * Queue a push of the terminal flip buffers to the line discipline. This
489 * function must not be called from IRQ context if port->low_latency is
492 * In the event of the queue being busy for flipping the work will be
493 * held off and retried later.
496 void tty_flip_buffer_push(struct tty_port *port)
498 struct tty_bufhead *buf = &port->buf;
500 buf->tail->commit = buf->tail->used;
502 if (port->low_latency)
503 flush_to_ldisc(&buf->work);
505 schedule_work(&buf->work);
507 EXPORT_SYMBOL(tty_flip_buffer_push);
510 * tty_buffer_init - prepare a tty buffer structure
511 * @tty: tty to initialise
513 * Set up the initial state of the buffer management for a tty device.
514 * Must be called before the other tty buffer functions are used.
517 void tty_buffer_init(struct tty_port *port)
519 struct tty_bufhead *buf = &port->buf;
521 mutex_init(&buf->lock);
522 tty_buffer_reset(&buf->sentinel, 0);
523 buf->head = &buf->sentinel;
524 buf->tail = &buf->sentinel;
525 init_llist_head(&buf->free);
526 atomic_set(&buf->memory_used, 0);
527 atomic_set(&buf->priority, 0);
528 INIT_WORK(&buf->work, flush_to_ldisc);