Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Tty buffer allocation management
4 : */
5 :
6 : #include <linux/types.h>
7 : #include <linux/errno.h>
8 : #include <linux/tty.h>
9 : #include <linux/tty_driver.h>
10 : #include <linux/tty_flip.h>
11 : #include <linux/timer.h>
12 : #include <linux/string.h>
13 : #include <linux/slab.h>
14 : #include <linux/sched.h>
15 : #include <linux/wait.h>
16 : #include <linux/bitops.h>
17 : #include <linux/delay.h>
18 : #include <linux/module.h>
19 : #include <linux/ratelimit.h>
20 :
21 :
22 : #define MIN_TTYB_SIZE 256
23 : #define TTYB_ALIGN_MASK 255
24 :
25 : /*
26 : * Byte threshold to limit memory consumption for flip buffers.
27 : * The actual memory limit is > 2x this amount.
28 : */
29 : #define TTYB_DEFAULT_MEM_LIMIT (640 * 1024UL)
30 :
31 : /*
32 : * We default to dicing tty buffer allocations to this many characters
33 : * in order to avoid multiple page allocations. We know the size of
34 : * tty_buffer itself but it must also be taken into account that the
35 : * the buffer is 256 byte aligned. See tty_buffer_find for the allocation
36 : * logic this must match
37 : */
38 :
39 : #define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~0xFF)
40 :
41 : /**
42 : * tty_buffer_lock_exclusive - gain exclusive access to buffer
43 : * tty_buffer_unlock_exclusive - release exclusive access
44 : *
45 : * @port: tty port owning the flip buffer
46 : *
47 : * Guarantees safe use of the line discipline's receive_buf() method by
48 : * excluding the buffer work and any pending flush from using the flip
49 : * buffer. Data can continue to be added concurrently to the flip buffer
50 : * from the driver side.
51 : *
52 : * On release, the buffer work is restarted if there is data in the
53 : * flip buffer
54 : */
55 :
56 0 : void tty_buffer_lock_exclusive(struct tty_port *port)
57 : {
58 0 : struct tty_bufhead *buf = &port->buf;
59 :
60 0 : atomic_inc(&buf->priority);
61 0 : mutex_lock(&buf->lock);
62 0 : }
63 : EXPORT_SYMBOL_GPL(tty_buffer_lock_exclusive);
64 :
65 0 : void tty_buffer_unlock_exclusive(struct tty_port *port)
66 : {
67 0 : struct tty_bufhead *buf = &port->buf;
68 0 : int restart;
69 :
70 0 : restart = buf->head->commit != buf->head->read;
71 :
72 0 : atomic_dec(&buf->priority);
73 0 : mutex_unlock(&buf->lock);
74 0 : if (restart)
75 0 : queue_work(system_unbound_wq, &buf->work);
76 0 : }
77 : EXPORT_SYMBOL_GPL(tty_buffer_unlock_exclusive);
78 :
79 : /**
80 : * tty_buffer_space_avail - return unused buffer space
81 : * @port: tty port owning the flip buffer
82 : *
83 : * Returns the # of bytes which can be written by the driver without
84 : * reaching the buffer limit.
85 : *
86 : * Note: this does not guarantee that memory is available to write
87 : * the returned # of bytes (use tty_prepare_flip_string_xxx() to
88 : * pre-allocate if memory guarantee is required).
89 : */
90 :
91 0 : int tty_buffer_space_avail(struct tty_port *port)
92 : {
93 0 : int space = port->buf.mem_limit - atomic_read(&port->buf.mem_used);
94 0 : return max(space, 0);
95 : }
96 : EXPORT_SYMBOL_GPL(tty_buffer_space_avail);
97 :
98 10 : static void tty_buffer_reset(struct tty_buffer *p, size_t size)
99 : {
100 10 : p->used = 0;
101 10 : p->size = size;
102 10 : p->next = NULL;
103 10 : p->commit = 0;
104 10 : p->read = 0;
105 10 : p->flags = 0;
106 : }
107 :
108 : /**
109 : * tty_buffer_free_all - free buffers used by a tty
110 : * @port: tty port to free from
111 : *
112 : * Remove all the buffers pending on a tty whether queued with data
113 : * or in the free ring. Must be called when the tty is no longer in use
114 : */
115 :
116 0 : void tty_buffer_free_all(struct tty_port *port)
117 : {
118 0 : struct tty_bufhead *buf = &port->buf;
119 0 : struct tty_buffer *p, *next;
120 0 : struct llist_node *llist;
121 0 : unsigned int freed = 0;
122 0 : int still_used;
123 :
124 0 : while ((p = buf->head) != NULL) {
125 0 : buf->head = p->next;
126 0 : freed += p->size;
127 0 : if (p->size > 0)
128 0 : kfree(p);
129 : }
130 0 : llist = llist_del_all(&buf->free);
131 0 : llist_for_each_entry_safe(p, next, llist, free)
132 0 : kfree(p);
133 :
134 0 : tty_buffer_reset(&buf->sentinel, 0);
135 0 : buf->head = &buf->sentinel;
136 0 : buf->tail = &buf->sentinel;
137 :
138 0 : still_used = atomic_xchg(&buf->mem_used, 0);
139 0 : WARN(still_used != freed, "we still have not freed %d bytes!",
140 : still_used - freed);
141 0 : }
142 :
143 : /**
144 : * tty_buffer_alloc - allocate a tty buffer
145 : * @port: tty port
146 : * @size: desired size (characters)
147 : *
148 : * Allocate a new tty buffer to hold the desired number of characters.
149 : * We round our buffers off in 256 character chunks to get better
150 : * allocation behaviour.
151 : * Return NULL if out of memory or the allocation would exceed the
152 : * per device queue
153 : */
154 :
155 0 : static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size)
156 : {
157 0 : struct llist_node *free;
158 0 : struct tty_buffer *p;
159 :
160 : /* Round the buffer size out */
161 0 : size = __ALIGN_MASK(size, TTYB_ALIGN_MASK);
162 :
163 0 : if (size <= MIN_TTYB_SIZE) {
164 0 : free = llist_del_first(&port->buf.free);
165 0 : if (free) {
166 0 : p = llist_entry(free, struct tty_buffer, free);
167 0 : goto found;
168 : }
169 : }
170 :
171 : /* Should possibly check if this fails for the largest buffer we
172 : have queued and recycle that ? */
173 0 : if (atomic_read(&port->buf.mem_used) > port->buf.mem_limit)
174 : return NULL;
175 0 : p = kmalloc(sizeof(struct tty_buffer) + 2 * size, GFP_ATOMIC);
176 0 : if (p == NULL)
177 : return NULL;
178 :
179 0 : found:
180 0 : tty_buffer_reset(p, size);
181 0 : atomic_add(size, &port->buf.mem_used);
182 0 : return p;
183 : }
184 :
185 : /**
186 : * tty_buffer_free - free a tty buffer
187 : * @port: tty port owning the buffer
188 : * @b: the buffer to free
189 : *
190 : * Free a tty buffer, or add it to the free list according to our
191 : * internal strategy
192 : */
193 :
194 0 : static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b)
195 : {
196 0 : struct tty_bufhead *buf = &port->buf;
197 :
198 : /* Dumb strategy for now - should keep some stats */
199 0 : WARN_ON(atomic_sub_return(b->size, &buf->mem_used) < 0);
200 :
201 0 : if (b->size > MIN_TTYB_SIZE)
202 0 : kfree(b);
203 0 : else if (b->size > 0)
204 0 : llist_add(&b->free, &buf->free);
205 0 : }
206 :
207 : /**
208 : * tty_buffer_flush - flush full tty buffers
209 : * @tty: tty to flush
210 : * @ld: optional ldisc ptr (must be referenced)
211 : *
212 : * flush all the buffers containing receive data. If ld != NULL,
213 : * flush the ldisc input buffer.
214 : *
215 : * Locking: takes buffer lock to ensure single-threaded flip buffer
216 : * 'consumer'
217 : */
218 :
219 282 : void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld)
220 : {
221 282 : struct tty_port *port = tty->port;
222 282 : struct tty_bufhead *buf = &port->buf;
223 282 : struct tty_buffer *next;
224 :
225 282 : atomic_inc(&buf->priority);
226 :
227 282 : mutex_lock(&buf->lock);
228 : /* paired w/ release in __tty_buffer_request_room; ensures there are
229 : * no pending memory accesses to the freed buffer
230 : */
231 282 : while ((next = smp_load_acquire(&buf->head->next)) != NULL) {
232 0 : tty_buffer_free(port, buf->head);
233 0 : buf->head = next;
234 : }
235 282 : buf->head->read = buf->head->commit;
236 :
237 282 : if (ld && ld->ops->flush_buffer)
238 279 : ld->ops->flush_buffer(tty);
239 :
240 282 : atomic_dec(&buf->priority);
241 282 : mutex_unlock(&buf->lock);
242 282 : }
243 :
244 : /**
245 : * tty_buffer_request_room - grow tty buffer if needed
246 : * @port: tty port
247 : * @size: size desired
248 : * @flags: buffer flags if new buffer allocated (default = 0)
249 : *
250 : * Make at least size bytes of linear space available for the tty
251 : * buffer. If we fail return the size we managed to find.
252 : *
253 : * Will change over to a new buffer if the current buffer is encoded as
254 : * TTY_NORMAL (so has no flags buffer) and the new buffer requires
255 : * a flags buffer.
256 : */
257 0 : static int __tty_buffer_request_room(struct tty_port *port, size_t size,
258 : int flags)
259 : {
260 0 : struct tty_bufhead *buf = &port->buf;
261 0 : struct tty_buffer *b, *n;
262 0 : int left, change;
263 :
264 0 : b = buf->tail;
265 0 : if (b->flags & TTYB_NORMAL)
266 0 : left = 2 * b->size - b->used;
267 : else
268 0 : left = b->size - b->used;
269 :
270 0 : change = (b->flags & TTYB_NORMAL) && (~flags & TTYB_NORMAL);
271 0 : if (change || left < size) {
272 : /* This is the slow path - looking for new buffers to use */
273 0 : n = tty_buffer_alloc(port, size);
274 0 : if (n != NULL) {
275 0 : n->flags = flags;
276 0 : buf->tail = n;
277 : /* paired w/ acquire in flush_to_ldisc(); ensures
278 : * flush_to_ldisc() sees buffer data.
279 : */
280 0 : smp_store_release(&b->commit, b->used);
281 : /* paired w/ acquire in flush_to_ldisc(); ensures the
282 : * latest commit value can be read before the head is
283 : * advanced to the next buffer
284 : */
285 0 : smp_store_release(&b->next, n);
286 0 : } else if (change)
287 : size = 0;
288 : else
289 0 : size = left;
290 : }
291 0 : return size;
292 : }
293 :
294 0 : int tty_buffer_request_room(struct tty_port *port, size_t size)
295 : {
296 0 : return __tty_buffer_request_room(port, size, 0);
297 : }
298 : EXPORT_SYMBOL_GPL(tty_buffer_request_room);
299 :
300 : /**
301 : * tty_insert_flip_string_fixed_flag - Add characters to the tty buffer
302 : * @port: tty port
303 : * @chars: characters
304 : * @flag: flag value for each character
305 : * @size: size
306 : *
307 : * Queue a series of bytes to the tty buffering. All the characters
308 : * passed are marked with the supplied flag. Returns the number added.
309 : */
310 :
311 0 : int tty_insert_flip_string_fixed_flag(struct tty_port *port,
312 : const unsigned char *chars, char flag, size_t size)
313 : {
314 0 : int copied = 0;
315 0 : do {
316 0 : int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
317 0 : int flags = (flag == TTY_NORMAL) ? TTYB_NORMAL : 0;
318 0 : int space = __tty_buffer_request_room(port, goal, flags);
319 0 : struct tty_buffer *tb = port->buf.tail;
320 0 : if (unlikely(space == 0))
321 : break;
322 0 : memcpy(char_buf_ptr(tb, tb->used), chars, space);
323 0 : if (~tb->flags & TTYB_NORMAL)
324 0 : memset(flag_buf_ptr(tb, tb->used), flag, space);
325 0 : tb->used += space;
326 0 : copied += space;
327 0 : chars += space;
328 : /* There is a small chance that we need to split the data over
329 : several buffers. If this is the case we must loop */
330 0 : } while (unlikely(size > copied));
331 0 : return copied;
332 : }
333 : EXPORT_SYMBOL(tty_insert_flip_string_fixed_flag);
334 :
335 : /**
336 : * tty_insert_flip_string_flags - Add characters to the tty buffer
337 : * @port: tty port
338 : * @chars: characters
339 : * @flags: flag bytes
340 : * @size: size
341 : *
342 : * Queue a series of bytes to the tty buffering. For each character
343 : * the flags array indicates the status of the character. Returns the
344 : * number added.
345 : */
346 :
347 0 : int tty_insert_flip_string_flags(struct tty_port *port,
348 : const unsigned char *chars, const char *flags, size_t size)
349 : {
350 0 : int copied = 0;
351 0 : do {
352 0 : int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
353 0 : int space = tty_buffer_request_room(port, goal);
354 0 : struct tty_buffer *tb = port->buf.tail;
355 0 : if (unlikely(space == 0))
356 : break;
357 0 : memcpy(char_buf_ptr(tb, tb->used), chars, space);
358 0 : memcpy(flag_buf_ptr(tb, tb->used), flags, space);
359 0 : tb->used += space;
360 0 : copied += space;
361 0 : chars += space;
362 0 : flags += space;
363 : /* There is a small chance that we need to split the data over
364 : several buffers. If this is the case we must loop */
365 0 : } while (unlikely(size > copied));
366 0 : return copied;
367 : }
368 : EXPORT_SYMBOL(tty_insert_flip_string_flags);
369 :
370 : /**
371 : * __tty_insert_flip_char - Add one character to the tty buffer
372 : * @port: tty port
373 : * @ch: character
374 : * @flag: flag byte
375 : *
376 : * Queue a single byte to the tty buffering, with an optional flag.
377 : * This is the slow path of tty_insert_flip_char.
378 : */
379 0 : int __tty_insert_flip_char(struct tty_port *port, unsigned char ch, char flag)
380 : {
381 0 : struct tty_buffer *tb;
382 0 : int flags = (flag == TTY_NORMAL) ? TTYB_NORMAL : 0;
383 :
384 0 : if (!__tty_buffer_request_room(port, 1, flags))
385 : return 0;
386 :
387 0 : tb = port->buf.tail;
388 0 : if (~tb->flags & TTYB_NORMAL)
389 0 : *flag_buf_ptr(tb, tb->used) = flag;
390 0 : *char_buf_ptr(tb, tb->used++) = ch;
391 :
392 0 : return 1;
393 : }
394 : EXPORT_SYMBOL(__tty_insert_flip_char);
395 :
396 : /**
397 : * tty_schedule_flip - push characters to ldisc
398 : * @port: tty port to push from
399 : *
400 : * Takes any pending buffers and transfers their ownership to the
401 : * ldisc side of the queue. It then schedules those characters for
402 : * processing by the line discipline.
403 : */
404 :
405 0 : void tty_schedule_flip(struct tty_port *port)
406 : {
407 0 : struct tty_bufhead *buf = &port->buf;
408 :
409 : /* paired w/ acquire in flush_to_ldisc(); ensures
410 : * flush_to_ldisc() sees buffer data.
411 : */
412 0 : smp_store_release(&buf->tail->commit, buf->tail->used);
413 0 : queue_work(system_unbound_wq, &buf->work);
414 0 : }
415 : EXPORT_SYMBOL(tty_schedule_flip);
416 :
417 : /**
418 : * tty_prepare_flip_string - make room for characters
419 : * @port: tty port
420 : * @chars: return pointer for character write area
421 : * @size: desired size
422 : *
423 : * Prepare a block of space in the buffer for data. Returns the length
424 : * available and buffer pointer to the space which is now allocated and
425 : * accounted for as ready for normal characters. This is used for drivers
426 : * that need their own block copy routines into the buffer. There is no
427 : * guarantee the buffer is a DMA target!
428 : */
429 :
430 0 : int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars,
431 : size_t size)
432 : {
433 0 : int space = __tty_buffer_request_room(port, size, TTYB_NORMAL);
434 0 : if (likely(space)) {
435 0 : struct tty_buffer *tb = port->buf.tail;
436 0 : *chars = char_buf_ptr(tb, tb->used);
437 0 : if (~tb->flags & TTYB_NORMAL)
438 0 : memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space);
439 0 : tb->used += space;
440 : }
441 0 : return space;
442 : }
443 : EXPORT_SYMBOL_GPL(tty_prepare_flip_string);
444 :
445 : /**
446 : * tty_ldisc_receive_buf - forward data to line discipline
447 : * @ld: line discipline to process input
448 : * @p: char buffer
449 : * @f: TTY_* flags buffer
450 : * @count: number of bytes to process
451 : *
452 : * Callers other than flush_to_ldisc() need to exclude the kworker
453 : * from concurrent use of the line discipline, see paste_selection().
454 : *
455 : * Returns the number of bytes processed
456 : */
457 0 : int tty_ldisc_receive_buf(struct tty_ldisc *ld, const unsigned char *p,
458 : char *f, int count)
459 : {
460 0 : if (ld->ops->receive_buf2)
461 0 : count = ld->ops->receive_buf2(ld->tty, p, f, count);
462 : else {
463 0 : count = min_t(int, count, ld->tty->receive_room);
464 0 : if (count && ld->ops->receive_buf)
465 0 : ld->ops->receive_buf(ld->tty, p, f, count);
466 : }
467 0 : return count;
468 : }
469 : EXPORT_SYMBOL_GPL(tty_ldisc_receive_buf);
470 :
471 : static int
472 0 : receive_buf(struct tty_port *port, struct tty_buffer *head, int count)
473 : {
474 0 : unsigned char *p = char_buf_ptr(head, head->read);
475 0 : char *f = NULL;
476 0 : int n;
477 :
478 0 : if (~head->flags & TTYB_NORMAL)
479 0 : f = flag_buf_ptr(head, head->read);
480 :
481 0 : n = port->client_ops->receive_buf(port, p, f, count);
482 0 : if (n > 0)
483 0 : memset(p, 0, n);
484 0 : return n;
485 : }
486 :
487 : /**
488 : * flush_to_ldisc
489 : * @work: tty structure passed from work queue.
490 : *
491 : * This routine is called out of the software interrupt to flush data
492 : * from the buffer chain to the line discipline.
493 : *
494 : * The receive_buf method is single threaded for each tty instance.
495 : *
496 : * Locking: takes buffer lock to ensure single-threaded flip buffer
497 : * 'consumer'
498 : */
499 :
500 0 : static void flush_to_ldisc(struct work_struct *work)
501 : {
502 0 : struct tty_port *port = container_of(work, struct tty_port, buf.work);
503 0 : struct tty_bufhead *buf = &port->buf;
504 :
505 0 : mutex_lock(&buf->lock);
506 :
507 0 : while (1) {
508 0 : struct tty_buffer *head = buf->head;
509 0 : struct tty_buffer *next;
510 0 : int count;
511 :
512 : /* Ldisc or user is trying to gain exclusive access */
513 0 : if (atomic_read(&buf->priority))
514 : break;
515 :
516 : /* paired w/ release in __tty_buffer_request_room();
517 : * ensures commit value read is not stale if the head
518 : * is advancing to the next buffer
519 : */
520 0 : next = smp_load_acquire(&head->next);
521 : /* paired w/ release in __tty_buffer_request_room() or in
522 : * tty_buffer_flush(); ensures we see the committed buffer data
523 : */
524 0 : count = smp_load_acquire(&head->commit) - head->read;
525 0 : if (!count) {
526 0 : if (next == NULL)
527 : break;
528 0 : buf->head = next;
529 0 : tty_buffer_free(port, head);
530 0 : continue;
531 : }
532 :
533 0 : count = receive_buf(port, head, count);
534 0 : if (!count)
535 : break;
536 0 : head->read += count;
537 : }
538 :
539 0 : mutex_unlock(&buf->lock);
540 :
541 0 : }
542 :
543 : /**
544 : * tty_flip_buffer_push - terminal
545 : * @port: tty port to push
546 : *
547 : * Queue a push of the terminal flip buffers to the line discipline.
548 : * Can be called from IRQ/atomic context.
549 : *
550 : * In the event of the queue being busy for flipping the work will be
551 : * held off and retried later.
552 : */
553 :
554 0 : void tty_flip_buffer_push(struct tty_port *port)
555 : {
556 0 : tty_schedule_flip(port);
557 0 : }
558 : EXPORT_SYMBOL(tty_flip_buffer_push);
559 :
560 : /**
561 : * tty_buffer_init - prepare a tty buffer structure
562 : * @port: tty port to initialise
563 : *
564 : * Set up the initial state of the buffer management for a tty device.
565 : * Must be called before the other tty buffer functions are used.
566 : */
567 :
568 10 : void tty_buffer_init(struct tty_port *port)
569 : {
570 10 : struct tty_bufhead *buf = &port->buf;
571 :
572 10 : mutex_init(&buf->lock);
573 10 : tty_buffer_reset(&buf->sentinel, 0);
574 10 : buf->head = &buf->sentinel;
575 10 : buf->tail = &buf->sentinel;
576 10 : init_llist_head(&buf->free);
577 10 : atomic_set(&buf->mem_used, 0);
578 10 : atomic_set(&buf->priority, 0);
579 10 : INIT_WORK(&buf->work, flush_to_ldisc);
580 10 : buf->mem_limit = TTYB_DEFAULT_MEM_LIMIT;
581 10 : }
582 :
583 : /**
584 : * tty_buffer_set_limit - change the tty buffer memory limit
585 : * @port: tty port to change
586 : * @limit: memory limit to set
587 : *
588 : * Change the tty buffer memory limit.
589 : * Must be called before the other tty buffer functions are used.
590 : */
591 :
592 0 : int tty_buffer_set_limit(struct tty_port *port, int limit)
593 : {
594 0 : if (limit < MIN_TTYB_SIZE)
595 : return -EINVAL;
596 0 : port->buf.mem_limit = limit;
597 0 : return 0;
598 : }
599 : EXPORT_SYMBOL_GPL(tty_buffer_set_limit);
600 :
601 : /* slave ptys can claim nested buffer lock when handling BRK and INTR */
602 0 : void tty_buffer_set_lock_subclass(struct tty_port *port)
603 : {
604 0 : lockdep_set_subclass(&port->buf.lock, TTY_LOCK_SLAVE);
605 0 : }
606 :
607 0 : bool tty_buffer_restart_work(struct tty_port *port)
608 : {
609 0 : return queue_work(system_unbound_wq, &port->buf.work);
610 : }
611 :
612 161 : bool tty_buffer_cancel_work(struct tty_port *port)
613 : {
614 161 : return cancel_work_sync(&port->buf.work);
615 : }
616 :
617 1 : void tty_buffer_flush_work(struct tty_port *port)
618 : {
619 1 : flush_work(&port->buf.work);
620 1 : }
|
