1 /**************************************************************************
3 * Copyright (c) 2009-2013 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
30 * While no substantial code is shared, the prime code is inspired by
33 * Dave Airlie <airlied@redhat.com>
34 * Rob Clark <rob.clark@linaro.org>
36 /** @file ttm_ref_object.c
38 * Base- and reference object implementation for the various
39 * ttm objects. Implements reference counting, minimal security checks
40 * and release on file close.
45 * struct ttm_object_file
47 * @tdev: Pointer to the ttm_object_device.
49 * @lock: Lock that protects the ref_list list and the
50 * ref_hash hash tables.
52 * @ref_list: List of ttm_ref_objects to be destroyed at
55 * @ref_hash: Hash tables of ref objects, one per ttm_ref_type,
56 * for fast lookup of ref objects given a base object.
59 #define pr_fmt(fmt) "[TTM] " fmt
61 #include <drm/ttm/ttm_object.h>
62 #include <drm/ttm/ttm_module.h>
63 #include <linux/list.h>
64 #include <linux/spinlock.h>
65 #include <linux/slab.h>
66 #include <linux/module.h>
67 #include <linux/atomic.h>
69 struct ttm_object_file {
70 struct ttm_object_device *tdev;
72 struct list_head ref_list;
73 struct drm_open_hash ref_hash[TTM_REF_NUM];
78 * struct ttm_object_device
80 * @object_lock: lock that protects the object_hash hash table.
82 * @object_hash: hash table for fast lookup of object global names.
84 * @object_count: Per device object count.
86 * This is the per-device data structure needed for ttm object management.
89 struct ttm_object_device {
90 spinlock_t object_lock;
91 struct drm_open_hash object_hash;
92 atomic_t object_count;
93 struct ttm_mem_global *mem_glob;
94 struct dma_buf_ops ops;
95 void (*dmabuf_release)(struct dma_buf *dma_buf);
100 * struct ttm_ref_object
102 * @hash: Hash entry for the per-file object reference hash.
104 * @head: List entry for the per-file list of ref-objects.
108 * @obj: Base object this ref object is referencing.
110 * @ref_type: Type of ref object.
112 * This is similar to an idr object, but it also has a hash table entry
113 * that allows lookup with a pointer to the referenced object as a key. In
114 * that way, one can easily detect whether a base object is referenced by
115 * a particular ttm_object_file. It also carries a ref count to avoid creating
116 * multiple ref objects if a ttm_object_file references the same base
117 * object more than once.
120 struct ttm_ref_object {
121 struct rcu_head rcu_head;
122 struct drm_hash_item hash;
123 struct list_head head;
125 enum ttm_ref_type ref_type;
126 struct ttm_base_object *obj;
127 struct ttm_object_file *tfile;
130 static void ttm_prime_dmabuf_release(struct dma_buf *dma_buf);
132 static inline struct ttm_object_file *
133 ttm_object_file_ref(struct ttm_object_file *tfile)
135 kref_get(&tfile->refcount);
139 static void ttm_object_file_destroy(struct kref *kref)
141 struct ttm_object_file *tfile =
142 container_of(kref, struct ttm_object_file, refcount);
148 static inline void ttm_object_file_unref(struct ttm_object_file **p_tfile)
150 struct ttm_object_file *tfile = *p_tfile;
153 kref_put(&tfile->refcount, ttm_object_file_destroy);
157 int ttm_base_object_init(struct ttm_object_file *tfile,
158 struct ttm_base_object *base,
160 enum ttm_object_type object_type,
161 void (*refcount_release) (struct ttm_base_object **),
162 void (*ref_obj_release) (struct ttm_base_object *,
163 enum ttm_ref_type ref_type))
165 struct ttm_object_device *tdev = tfile->tdev;
168 base->shareable = shareable;
169 base->tfile = ttm_object_file_ref(tfile);
170 base->refcount_release = refcount_release;
171 base->ref_obj_release = ref_obj_release;
172 base->object_type = object_type;
173 kref_init(&base->refcount);
174 spin_lock(&tdev->object_lock);
175 ret = drm_ht_just_insert_please_rcu(&tdev->object_hash,
177 (unsigned long)base, 31, 0, 0);
178 spin_unlock(&tdev->object_lock);
179 if (unlikely(ret != 0))
182 ret = ttm_ref_object_add(tfile, base, TTM_REF_USAGE, NULL);
183 if (unlikely(ret != 0))
186 ttm_base_object_unref(&base);
190 spin_lock(&tdev->object_lock);
191 (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash);
192 spin_unlock(&tdev->object_lock);
196 EXPORT_SYMBOL(ttm_base_object_init);
198 static void ttm_release_base(struct kref *kref)
200 struct ttm_base_object *base =
201 container_of(kref, struct ttm_base_object, refcount);
202 struct ttm_object_device *tdev = base->tfile->tdev;
204 spin_lock(&tdev->object_lock);
205 (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash);
206 spin_unlock(&tdev->object_lock);
209 * Note: We don't use synchronize_rcu() here because it's far
210 * too slow. It's up to the user to free the object using
211 * call_rcu() or ttm_base_object_kfree().
214 ttm_object_file_unref(&base->tfile);
215 if (base->refcount_release)
216 base->refcount_release(&base);
219 void ttm_base_object_unref(struct ttm_base_object **p_base)
221 struct ttm_base_object *base = *p_base;
225 kref_put(&base->refcount, ttm_release_base);
227 EXPORT_SYMBOL(ttm_base_object_unref);
229 struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file *tfile,
232 struct ttm_base_object *base = NULL;
233 struct drm_hash_item *hash;
234 struct drm_open_hash *ht = &tfile->ref_hash[TTM_REF_USAGE];
238 ret = drm_ht_find_item_rcu(ht, key, &hash);
240 if (likely(ret == 0)) {
241 base = drm_hash_entry(hash, struct ttm_ref_object, hash)->obj;
242 if (!kref_get_unless_zero(&base->refcount))
249 EXPORT_SYMBOL(ttm_base_object_lookup);
251 struct ttm_base_object *
252 ttm_base_object_lookup_for_ref(struct ttm_object_device *tdev, uint32_t key)
254 struct ttm_base_object *base = NULL;
255 struct drm_hash_item *hash;
256 struct drm_open_hash *ht = &tdev->object_hash;
260 ret = drm_ht_find_item_rcu(ht, key, &hash);
262 if (likely(ret == 0)) {
263 base = drm_hash_entry(hash, struct ttm_base_object, hash);
264 if (!kref_get_unless_zero(&base->refcount))
271 EXPORT_SYMBOL(ttm_base_object_lookup_for_ref);
273 int ttm_ref_object_add(struct ttm_object_file *tfile,
274 struct ttm_base_object *base,
275 enum ttm_ref_type ref_type, bool *existed)
277 struct drm_open_hash *ht = &tfile->ref_hash[ref_type];
278 struct ttm_ref_object *ref;
279 struct drm_hash_item *hash;
280 struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob;
283 if (base->tfile != tfile && !base->shareable)
289 while (ret == -EINVAL) {
291 ret = drm_ht_find_item_rcu(ht, base->hash.key, &hash);
294 ref = drm_hash_entry(hash, struct ttm_ref_object, hash);
295 if (!kref_get_unless_zero(&ref->kref)) {
302 ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref),
304 if (unlikely(ret != 0))
306 ref = kmalloc(sizeof(*ref), GFP_KERNEL);
307 if (unlikely(ref == NULL)) {
308 ttm_mem_global_free(mem_glob, sizeof(*ref));
312 ref->hash.key = base->hash.key;
315 ref->ref_type = ref_type;
316 kref_init(&ref->kref);
318 spin_lock(&tfile->lock);
319 ret = drm_ht_insert_item_rcu(ht, &ref->hash);
321 if (likely(ret == 0)) {
322 list_add_tail(&ref->head, &tfile->ref_list);
323 kref_get(&base->refcount);
324 spin_unlock(&tfile->lock);
330 spin_unlock(&tfile->lock);
331 BUG_ON(ret != -EINVAL);
333 ttm_mem_global_free(mem_glob, sizeof(*ref));
339 EXPORT_SYMBOL(ttm_ref_object_add);
341 static void ttm_ref_object_release(struct kref *kref)
343 struct ttm_ref_object *ref =
344 container_of(kref, struct ttm_ref_object, kref);
345 struct ttm_base_object *base = ref->obj;
346 struct ttm_object_file *tfile = ref->tfile;
347 struct drm_open_hash *ht;
348 struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob;
350 ht = &tfile->ref_hash[ref->ref_type];
351 (void)drm_ht_remove_item_rcu(ht, &ref->hash);
352 list_del(&ref->head);
353 spin_unlock(&tfile->lock);
355 if (ref->ref_type != TTM_REF_USAGE && base->ref_obj_release)
356 base->ref_obj_release(base, ref->ref_type);
358 ttm_base_object_unref(&ref->obj);
359 ttm_mem_global_free(mem_glob, sizeof(*ref));
360 kfree_rcu(ref, rcu_head);
361 spin_lock(&tfile->lock);
364 int ttm_ref_object_base_unref(struct ttm_object_file *tfile,
365 unsigned long key, enum ttm_ref_type ref_type)
367 struct drm_open_hash *ht = &tfile->ref_hash[ref_type];
368 struct ttm_ref_object *ref;
369 struct drm_hash_item *hash;
372 spin_lock(&tfile->lock);
373 ret = drm_ht_find_item(ht, key, &hash);
374 if (unlikely(ret != 0)) {
375 spin_unlock(&tfile->lock);
378 ref = drm_hash_entry(hash, struct ttm_ref_object, hash);
379 kref_put(&ref->kref, ttm_ref_object_release);
380 spin_unlock(&tfile->lock);
383 EXPORT_SYMBOL(ttm_ref_object_base_unref);
385 void ttm_object_file_release(struct ttm_object_file **p_tfile)
387 struct ttm_ref_object *ref;
388 struct list_head *list;
390 struct ttm_object_file *tfile = *p_tfile;
393 spin_lock(&tfile->lock);
396 * Since we release the lock within the loop, we have to
397 * restart it from the beginning each time.
400 while (!list_empty(&tfile->ref_list)) {
401 list = tfile->ref_list.next;
402 ref = list_entry(list, struct ttm_ref_object, head);
403 ttm_ref_object_release(&ref->kref);
406 for (i = 0; i < TTM_REF_NUM; ++i)
407 drm_ht_remove(&tfile->ref_hash[i]);
409 spin_unlock(&tfile->lock);
410 ttm_object_file_unref(&tfile);
412 EXPORT_SYMBOL(ttm_object_file_release);
414 struct ttm_object_file *ttm_object_file_init(struct ttm_object_device *tdev,
415 unsigned int hash_order)
417 struct ttm_object_file *tfile = kmalloc(sizeof(*tfile), GFP_KERNEL);
422 if (unlikely(tfile == NULL))
425 spin_lock_init(&tfile->lock);
427 kref_init(&tfile->refcount);
428 INIT_LIST_HEAD(&tfile->ref_list);
430 for (i = 0; i < TTM_REF_NUM; ++i) {
431 ret = drm_ht_create(&tfile->ref_hash[i], hash_order);
440 for (i = 0; i < j; ++i)
441 drm_ht_remove(&tfile->ref_hash[i]);
447 EXPORT_SYMBOL(ttm_object_file_init);
449 struct ttm_object_device *
450 ttm_object_device_init(struct ttm_mem_global *mem_glob,
451 unsigned int hash_order,
452 const struct dma_buf_ops *ops)
454 struct ttm_object_device *tdev = kmalloc(sizeof(*tdev), GFP_KERNEL);
457 if (unlikely(tdev == NULL))
460 tdev->mem_glob = mem_glob;
461 spin_lock_init(&tdev->object_lock);
462 atomic_set(&tdev->object_count, 0);
463 ret = drm_ht_create(&tdev->object_hash, hash_order);
465 goto out_no_object_hash;
468 tdev->dmabuf_release = tdev->ops.release;
469 tdev->ops.release = ttm_prime_dmabuf_release;
470 tdev->dma_buf_size = ttm_round_pot(sizeof(struct dma_buf)) +
471 ttm_round_pot(sizeof(struct file));
478 EXPORT_SYMBOL(ttm_object_device_init);
480 void ttm_object_device_release(struct ttm_object_device **p_tdev)
482 struct ttm_object_device *tdev = *p_tdev;
486 spin_lock(&tdev->object_lock);
487 drm_ht_remove(&tdev->object_hash);
488 spin_unlock(&tdev->object_lock);
492 EXPORT_SYMBOL(ttm_object_device_release);
495 * get_dma_buf_unless_doomed - get a dma_buf reference if possible.
497 * @dma_buf: Non-refcounted pointer to a struct dma-buf.
499 * Obtain a file reference from a lookup structure that doesn't refcount
500 * the file, but synchronizes with its release method to make sure it has
501 * not been freed yet. See for example kref_get_unless_zero documentation.
502 * Returns true if refcounting succeeds, false otherwise.
504 * Nobody really wants this as a public API yet, so let it mature here
507 static bool __must_check get_dma_buf_unless_doomed(struct dma_buf *dmabuf)
509 return atomic_long_inc_not_zero(&dmabuf->file->f_count) != 0L;
513 * ttm_prime_refcount_release - refcount release method for a prime object.
515 * @p_base: Pointer to ttm_base_object pointer.
517 * This is a wrapper that calls the refcount_release founction of the
518 * underlying object. At the same time it cleans up the prime object.
519 * This function is called when all references to the base object we
520 * derive from are gone.
522 static void ttm_prime_refcount_release(struct ttm_base_object **p_base)
524 struct ttm_base_object *base = *p_base;
525 struct ttm_prime_object *prime;
528 prime = container_of(base, struct ttm_prime_object, base);
529 BUG_ON(prime->dma_buf != NULL);
530 mutex_destroy(&prime->mutex);
531 if (prime->refcount_release)
532 prime->refcount_release(&base);
536 * ttm_prime_dmabuf_release - Release method for the dma-bufs we export
540 * This function first calls the dma_buf release method the driver
541 * provides. Then it cleans up our dma_buf pointer used for lookup,
542 * and finally releases the reference the dma_buf has on our base
545 static void ttm_prime_dmabuf_release(struct dma_buf *dma_buf)
547 struct ttm_prime_object *prime =
548 (struct ttm_prime_object *) dma_buf->priv;
549 struct ttm_base_object *base = &prime->base;
550 struct ttm_object_device *tdev = base->tfile->tdev;
552 if (tdev->dmabuf_release)
553 tdev->dmabuf_release(dma_buf);
554 mutex_lock(&prime->mutex);
555 if (prime->dma_buf == dma_buf)
556 prime->dma_buf = NULL;
557 mutex_unlock(&prime->mutex);
558 ttm_mem_global_free(tdev->mem_glob, tdev->dma_buf_size);
559 ttm_base_object_unref(&base);
563 * ttm_prime_fd_to_handle - Get a base object handle from a prime fd
565 * @tfile: A struct ttm_object_file identifying the caller.
566 * @fd: The prime / dmabuf fd.
567 * @handle: The returned handle.
569 * This function returns a handle to an object that previously exported
570 * a dma-buf. Note that we don't handle imports yet, because we simply
571 * have no consumers of that implementation.
573 int ttm_prime_fd_to_handle(struct ttm_object_file *tfile,
576 struct ttm_object_device *tdev = tfile->tdev;
577 struct dma_buf *dma_buf;
578 struct ttm_prime_object *prime;
579 struct ttm_base_object *base;
582 dma_buf = dma_buf_get(fd);
584 return PTR_ERR(dma_buf);
586 if (dma_buf->ops != &tdev->ops)
589 prime = (struct ttm_prime_object *) dma_buf->priv;
591 *handle = base->hash.key;
592 ret = ttm_ref_object_add(tfile, base, TTM_REF_USAGE, NULL);
594 dma_buf_put(dma_buf);
598 EXPORT_SYMBOL_GPL(ttm_prime_fd_to_handle);
601 * ttm_prime_handle_to_fd - Return a dma_buf fd from a ttm prime object
603 * @tfile: Struct ttm_object_file identifying the caller.
604 * @handle: Handle to the object we're exporting from.
605 * @flags: flags for dma-buf creation. We just pass them on.
606 * @prime_fd: The returned file descriptor.
609 int ttm_prime_handle_to_fd(struct ttm_object_file *tfile,
610 uint32_t handle, uint32_t flags,
613 struct ttm_object_device *tdev = tfile->tdev;
614 struct ttm_base_object *base;
615 struct dma_buf *dma_buf;
616 struct ttm_prime_object *prime;
619 base = ttm_base_object_lookup(tfile, handle);
620 if (unlikely(base == NULL ||
621 base->object_type != ttm_prime_type)) {
626 prime = container_of(base, struct ttm_prime_object, base);
627 if (unlikely(!base->shareable)) {
632 ret = mutex_lock_interruptible(&prime->mutex);
633 if (unlikely(ret != 0)) {
638 dma_buf = prime->dma_buf;
639 if (!dma_buf || !get_dma_buf_unless_doomed(dma_buf)) {
642 * Need to create a new dma_buf, with memory accounting.
644 ret = ttm_mem_global_alloc(tdev->mem_glob, tdev->dma_buf_size,
646 if (unlikely(ret != 0)) {
647 mutex_unlock(&prime->mutex);
651 dma_buf = dma_buf_export(prime, &tdev->ops,
653 if (IS_ERR(dma_buf)) {
654 ret = PTR_ERR(dma_buf);
655 ttm_mem_global_free(tdev->mem_glob,
657 mutex_unlock(&prime->mutex);
662 * dma_buf has taken the base object reference
665 prime->dma_buf = dma_buf;
667 mutex_unlock(&prime->mutex);
669 ret = dma_buf_fd(dma_buf, flags);
674 dma_buf_put(dma_buf);
678 ttm_base_object_unref(&base);
681 EXPORT_SYMBOL_GPL(ttm_prime_handle_to_fd);
684 * ttm_prime_object_init - Initialize a ttm_prime_object
686 * @tfile: struct ttm_object_file identifying the caller
687 * @size: The size of the dma_bufs we export.
688 * @prime: The object to be initialized.
689 * @shareable: See ttm_base_object_init
690 * @type: See ttm_base_object_init
691 * @refcount_release: See ttm_base_object_init
692 * @ref_obj_release: See ttm_base_object_init
694 * Initializes an object which is compatible with the drm_prime model
695 * for data sharing between processes and devices.
697 int ttm_prime_object_init(struct ttm_object_file *tfile, size_t size,
698 struct ttm_prime_object *prime, bool shareable,
699 enum ttm_object_type type,
700 void (*refcount_release) (struct ttm_base_object **),
701 void (*ref_obj_release) (struct ttm_base_object *,
702 enum ttm_ref_type ref_type))
704 mutex_init(&prime->mutex);
705 prime->size = PAGE_ALIGN(size);
706 prime->real_type = type;
707 prime->dma_buf = NULL;
708 prime->refcount_release = refcount_release;
709 return ttm_base_object_init(tfile, &prime->base, shareable,
711 ttm_prime_refcount_release,
714 EXPORT_SYMBOL(ttm_prime_object_init);