4 * COMEDI - Linux Control and Measurement Device Interface
5 * Copyright (C) 1997-2000 David A. Schleef <ds@schleef.org>
6 * Copyright (C) 2002 Frank Mori Hess <fmhess@users.sourceforge.net>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
19 #include <linux/vmalloc.h>
20 #include <linux/slab.h>
22 #include "comedidev.h"
23 #include "comedi_internal.h"
25 #ifdef PAGE_KERNEL_NOCACHE
26 #define COMEDI_PAGE_PROTECTION PAGE_KERNEL_NOCACHE
28 #define COMEDI_PAGE_PROTECTION PAGE_KERNEL
31 static void comedi_buf_map_kref_release(struct kref *kref)
33 struct comedi_buf_map *bm =
34 container_of(kref, struct comedi_buf_map, refcount);
35 struct comedi_buf_page *buf;
39 for (i = 0; i < bm->n_pages; i++) {
40 buf = &bm->page_list[i];
41 clear_bit(PG_reserved,
42 &(virt_to_page(buf->virt_addr)->flags));
43 if (bm->dma_dir != DMA_NONE) {
45 dma_free_coherent(bm->dma_hw_dev,
51 free_page((unsigned long)buf->virt_addr);
56 if (bm->dma_dir != DMA_NONE)
57 put_device(bm->dma_hw_dev);
61 static void __comedi_buf_free(struct comedi_device *dev,
62 struct comedi_subdevice *s)
64 struct comedi_async *async = s->async;
65 struct comedi_buf_map *bm;
68 if (async->prealloc_buf) {
69 vunmap(async->prealloc_buf);
70 async->prealloc_buf = NULL;
71 async->prealloc_bufsz = 0;
74 spin_lock_irqsave(&s->spin_lock, flags);
76 async->buf_map = NULL;
77 spin_unlock_irqrestore(&s->spin_lock, flags);
78 comedi_buf_map_put(bm);
81 static void __comedi_buf_alloc(struct comedi_device *dev,
82 struct comedi_subdevice *s,
85 struct comedi_async *async = s->async;
86 struct page **pages = NULL;
87 struct comedi_buf_map *bm;
88 struct comedi_buf_page *buf;
92 if (!IS_ENABLED(CONFIG_HAS_DMA) && s->async_dma_dir != DMA_NONE) {
93 dev_err(dev->class_dev,
94 "dma buffer allocation not supported\n");
98 bm = kzalloc(sizeof(*async->buf_map), GFP_KERNEL);
102 kref_init(&bm->refcount);
103 spin_lock_irqsave(&s->spin_lock, flags);
105 spin_unlock_irqrestore(&s->spin_lock, flags);
106 bm->dma_dir = s->async_dma_dir;
107 if (bm->dma_dir != DMA_NONE)
108 /* Need ref to hardware device to free buffer later. */
109 bm->dma_hw_dev = get_device(dev->hw_dev);
111 bm->page_list = vzalloc(sizeof(*buf) * n_pages);
113 pages = vmalloc(sizeof(struct page *) * n_pages);
118 for (i = 0; i < n_pages; i++) {
119 buf = &bm->page_list[i];
120 if (bm->dma_dir != DMA_NONE)
121 #ifdef CONFIG_HAS_DMA
122 buf->virt_addr = dma_alloc_coherent(bm->dma_hw_dev,
131 buf->virt_addr = (void *)get_zeroed_page(GFP_KERNEL);
135 set_bit(PG_reserved, &(virt_to_page(buf->virt_addr)->flags));
137 pages[i] = virt_to_page(buf->virt_addr);
139 spin_lock_irqsave(&s->spin_lock, flags);
141 spin_unlock_irqrestore(&s->spin_lock, flags);
143 /* vmap the prealloc_buf if all the pages were allocated */
145 async->prealloc_buf = vmap(pages, n_pages, VM_MAP,
146 COMEDI_PAGE_PROTECTION);
151 void comedi_buf_map_get(struct comedi_buf_map *bm)
154 kref_get(&bm->refcount);
157 int comedi_buf_map_put(struct comedi_buf_map *bm)
160 return kref_put(&bm->refcount, comedi_buf_map_kref_release);
164 /* returns s->async->buf_map and increments its kref refcount */
165 struct comedi_buf_map *
166 comedi_buf_map_from_subdev_get(struct comedi_subdevice *s)
168 struct comedi_async *async = s->async;
169 struct comedi_buf_map *bm = NULL;
175 spin_lock_irqsave(&s->spin_lock, flags);
177 /* only want it if buffer pages allocated */
178 if (bm && bm->n_pages)
179 comedi_buf_map_get(bm);
182 spin_unlock_irqrestore(&s->spin_lock, flags);
187 bool comedi_buf_is_mmapped(struct comedi_subdevice *s)
189 struct comedi_buf_map *bm = s->async->buf_map;
191 return bm && (atomic_read(&bm->refcount.refcount) > 1);
194 int comedi_buf_alloc(struct comedi_device *dev, struct comedi_subdevice *s,
195 unsigned long new_size)
197 struct comedi_async *async = s->async;
199 /* Round up new_size to multiple of PAGE_SIZE */
200 new_size = (new_size + PAGE_SIZE - 1) & PAGE_MASK;
202 /* if no change is required, do nothing */
203 if (async->prealloc_buf && async->prealloc_bufsz == new_size)
206 /* deallocate old buffer */
207 __comedi_buf_free(dev, s);
209 /* allocate new buffer */
211 unsigned n_pages = new_size >> PAGE_SHIFT;
213 __comedi_buf_alloc(dev, s, n_pages);
215 if (!async->prealloc_buf) {
216 /* allocation failed */
217 __comedi_buf_free(dev, s);
221 async->prealloc_bufsz = new_size;
226 void comedi_buf_reset(struct comedi_subdevice *s)
228 struct comedi_async *async = s->async;
230 async->buf_write_alloc_count = 0;
231 async->buf_write_count = 0;
232 async->buf_read_alloc_count = 0;
233 async->buf_read_count = 0;
235 async->buf_write_ptr = 0;
236 async->buf_read_ptr = 0;
239 async->scan_progress = 0;
240 async->munge_chan = 0;
241 async->munge_count = 0;
242 async->munge_ptr = 0;
247 static unsigned int comedi_buf_write_n_available(struct comedi_subdevice *s)
249 struct comedi_async *async = s->async;
250 unsigned int free_end = async->buf_read_count + async->prealloc_bufsz;
252 return free_end - async->buf_write_alloc_count;
255 static unsigned int __comedi_buf_write_alloc(struct comedi_subdevice *s,
259 struct comedi_async *async = s->async;
260 unsigned int available = comedi_buf_write_n_available(s);
262 if (nbytes > available)
263 nbytes = strict ? 0 : available;
265 async->buf_write_alloc_count += nbytes;
268 * ensure the async buffer 'counts' are read and updated
269 * before we write data to the write-alloc'ed buffer space
276 /* allocates chunk for the writer from free buffer space */
277 unsigned int comedi_buf_write_alloc(struct comedi_subdevice *s,
280 return __comedi_buf_write_alloc(s, nbytes, 0);
282 EXPORT_SYMBOL_GPL(comedi_buf_write_alloc);
285 * munging is applied to data by core as it passes between user
288 static unsigned int comedi_buf_munge(struct comedi_subdevice *s,
289 unsigned int num_bytes)
291 struct comedi_async *async = s->async;
292 unsigned int count = 0;
293 const unsigned num_sample_bytes = bytes_per_sample(s);
295 if (!s->munge || (async->cmd.flags & CMDF_RAWDATA)) {
296 async->munge_count += num_bytes;
299 /* don't munge partial samples */
300 num_bytes -= num_bytes % num_sample_bytes;
301 while (count < num_bytes) {
302 int block_size = num_bytes - count;
303 unsigned int buf_end;
305 buf_end = async->prealloc_bufsz - async->munge_ptr;
306 if (block_size > buf_end)
307 block_size = buf_end;
309 s->munge(s->device, s,
310 async->prealloc_buf + async->munge_ptr,
311 block_size, async->munge_chan);
314 * ensure data is munged in buffer before the
315 * async buffer munge_count is incremented
319 async->munge_chan += block_size / num_sample_bytes;
320 async->munge_chan %= async->cmd.chanlist_len;
321 async->munge_count += block_size;
322 async->munge_ptr += block_size;
323 async->munge_ptr %= async->prealloc_bufsz;
331 unsigned int comedi_buf_write_n_allocated(struct comedi_subdevice *s)
333 struct comedi_async *async = s->async;
335 return async->buf_write_alloc_count - async->buf_write_count;
338 /* transfers a chunk from writer to filled buffer space */
339 unsigned int comedi_buf_write_free(struct comedi_subdevice *s,
342 struct comedi_async *async = s->async;
343 unsigned int allocated = comedi_buf_write_n_allocated(s);
345 if (nbytes > allocated)
348 async->buf_write_count += nbytes;
349 async->buf_write_ptr += nbytes;
350 comedi_buf_munge(s, async->buf_write_count - async->munge_count);
351 if (async->buf_write_ptr >= async->prealloc_bufsz)
352 async->buf_write_ptr %= async->prealloc_bufsz;
356 EXPORT_SYMBOL_GPL(comedi_buf_write_free);
358 unsigned int comedi_buf_read_n_available(struct comedi_subdevice *s)
360 struct comedi_async *async = s->async;
366 num_bytes = async->munge_count - async->buf_read_count;
369 * ensure the async buffer 'counts' are read before we
370 * attempt to read data from the buffer
376 EXPORT_SYMBOL_GPL(comedi_buf_read_n_available);
378 /* allocates a chunk for the reader from filled (and munged) buffer space */
379 unsigned int comedi_buf_read_alloc(struct comedi_subdevice *s,
382 struct comedi_async *async = s->async;
383 unsigned int available;
385 available = async->munge_count - async->buf_read_alloc_count;
386 if (nbytes > available)
389 async->buf_read_alloc_count += nbytes;
392 * ensure the async buffer 'counts' are read before we
393 * attempt to read data from the read-alloc'ed buffer space
399 EXPORT_SYMBOL_GPL(comedi_buf_read_alloc);
401 static unsigned int comedi_buf_read_n_allocated(struct comedi_async *async)
403 return async->buf_read_alloc_count - async->buf_read_count;
406 /* transfers control of a chunk from reader to free buffer space */
407 unsigned int comedi_buf_read_free(struct comedi_subdevice *s,
410 struct comedi_async *async = s->async;
411 unsigned int allocated;
414 * ensure data has been read out of buffer before
415 * the async read count is incremented
419 allocated = comedi_buf_read_n_allocated(async);
420 if (nbytes > allocated)
423 async->buf_read_count += nbytes;
424 async->buf_read_ptr += nbytes;
425 async->buf_read_ptr %= async->prealloc_bufsz;
428 EXPORT_SYMBOL_GPL(comedi_buf_read_free);
430 int comedi_buf_put(struct comedi_subdevice *s, unsigned short x)
432 struct comedi_async *async = s->async;
433 unsigned int n = __comedi_buf_write_alloc(s, sizeof(short), 1);
435 if (n < sizeof(short)) {
436 async->events |= COMEDI_CB_ERROR;
439 *(unsigned short *)(async->prealloc_buf + async->buf_write_ptr) = x;
440 comedi_buf_write_free(s, sizeof(short));
443 EXPORT_SYMBOL_GPL(comedi_buf_put);
445 int comedi_buf_get(struct comedi_subdevice *s, unsigned short *x)
447 struct comedi_async *async = s->async;
448 unsigned int n = comedi_buf_read_n_available(s);
450 if (n < sizeof(short))
452 comedi_buf_read_alloc(s, sizeof(short));
453 *x = *(unsigned short *)(async->prealloc_buf + async->buf_read_ptr);
454 comedi_buf_read_free(s, sizeof(short));
457 EXPORT_SYMBOL_GPL(comedi_buf_get);
459 void comedi_buf_memcpy_to(struct comedi_subdevice *s, unsigned int offset,
460 const void *data, unsigned int num_bytes)
462 struct comedi_async *async = s->async;
463 unsigned int write_ptr = async->buf_write_ptr + offset;
465 if (write_ptr >= async->prealloc_bufsz)
466 write_ptr %= async->prealloc_bufsz;
469 unsigned int block_size;
471 if (write_ptr + num_bytes > async->prealloc_bufsz)
472 block_size = async->prealloc_bufsz - write_ptr;
474 block_size = num_bytes;
476 memcpy(async->prealloc_buf + write_ptr, data, block_size);
479 num_bytes -= block_size;
484 EXPORT_SYMBOL_GPL(comedi_buf_memcpy_to);
486 void comedi_buf_memcpy_from(struct comedi_subdevice *s, unsigned int offset,
487 void *dest, unsigned int nbytes)
490 struct comedi_async *async = s->async;
491 unsigned int read_ptr = async->buf_read_ptr + offset;
493 if (read_ptr >= async->prealloc_bufsz)
494 read_ptr %= async->prealloc_bufsz;
497 unsigned int block_size;
499 src = async->prealloc_buf + read_ptr;
501 if (nbytes >= async->prealloc_bufsz - read_ptr)
502 block_size = async->prealloc_bufsz - read_ptr;
506 memcpy(dest, src, block_size);
507 nbytes -= block_size;
512 EXPORT_SYMBOL_GPL(comedi_buf_memcpy_from);
515 * comedi_write_array_to_buffer - write data to comedi buffer
516 * @s: comedi_subdevice struct
518 * @num_bytes: number of bytes to write
520 * Writes up to num_bytes bytes of data to the comedi buffer associated with
521 * the subdevice, marks it as written and updates the acquisition scan
524 * Returns the amount of data written in bytes.
526 unsigned int comedi_write_array_to_buffer(struct comedi_subdevice *s,
528 unsigned int num_bytes)
530 struct comedi_async *async = s->async;
536 retval = comedi_buf_write_alloc(s, num_bytes);
537 if (retval != num_bytes) {
538 dev_warn(s->device->class_dev, "buffer overrun\n");
539 async->events |= COMEDI_CB_OVERFLOW;
543 comedi_buf_memcpy_to(s, 0, data, num_bytes);
544 comedi_buf_write_free(s, num_bytes);
545 comedi_inc_scan_progress(s, num_bytes);
546 async->events |= COMEDI_CB_BLOCK;
550 EXPORT_SYMBOL_GPL(comedi_write_array_to_buffer);
553 * comedi_read_array_from_buffer - read data from comedi buffer
554 * @s: comedi_subdevice struct
556 * @num_bytes: number of bytes to read
558 * Reads up to num_bytes bytes of data from the comedi buffer associated with
559 * the subdevice, marks it as read and updates the acquisition scan progress.
561 * Returns the amount of data read in bytes.
563 unsigned int comedi_read_array_from_buffer(struct comedi_subdevice *s,
564 void *data, unsigned int num_bytes)
569 num_bytes = comedi_buf_read_alloc(s, num_bytes);
570 comedi_buf_memcpy_from(s, 0, data, num_bytes);
571 comedi_buf_read_free(s, num_bytes);
572 comedi_inc_scan_progress(s, num_bytes);
573 s->async->events |= COMEDI_CB_BLOCK;
577 EXPORT_SYMBOL_GPL(comedi_read_array_from_buffer);