2 * Copyright (C) 2007 Ben Skeggs.
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29 #include <linux/ktime.h>
30 #include <linux/hrtimer.h>
32 #include <nvif/notify.h>
33 #include <nvif/event.h>
35 #include "nouveau_drm.h"
36 #include "nouveau_dma.h"
37 #include "nouveau_fence.h"
40 struct work_struct base;
41 struct list_head head;
47 nouveau_fence_signal(struct nouveau_fence *fence)
49 struct fence_work *work, *temp;
51 list_for_each_entry_safe(work, temp, &fence->work, head) {
52 schedule_work(&work->base);
53 list_del(&work->head);
56 fence->channel = NULL;
57 list_del(&fence->head);
61 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
63 struct nouveau_fence *fence, *fnext;
64 spin_lock(&fctx->lock);
65 list_for_each_entry_safe(fence, fnext, &fctx->pending, head) {
66 nouveau_fence_signal(fence);
68 spin_unlock(&fctx->lock);
72 nouveau_fence_context_new(struct nouveau_fence_chan *fctx)
74 INIT_LIST_HEAD(&fctx->flip);
75 INIT_LIST_HEAD(&fctx->pending);
76 spin_lock_init(&fctx->lock);
80 nouveau_fence_work_handler(struct work_struct *kwork)
82 struct fence_work *work = container_of(kwork, typeof(*work), base);
83 work->func(work->data);
88 nouveau_fence_work(struct nouveau_fence *fence,
89 void (*func)(void *), void *data)
91 struct nouveau_channel *chan = fence->channel;
92 struct nouveau_fence_chan *fctx;
93 struct fence_work *work = NULL;
95 if (nouveau_fence_done(fence)) {
101 work = kmalloc(sizeof(*work), GFP_KERNEL);
103 WARN_ON(nouveau_fence_wait(fence, false, false));
108 spin_lock(&fctx->lock);
109 if (!fence->channel) {
110 spin_unlock(&fctx->lock);
116 INIT_WORK(&work->base, nouveau_fence_work_handler);
119 list_add(&work->head, &fence->work);
120 spin_unlock(&fctx->lock);
124 nouveau_fence_update(struct nouveau_channel *chan)
126 struct nouveau_fence_chan *fctx = chan->fence;
127 struct nouveau_fence *fence, *fnext;
129 spin_lock(&fctx->lock);
130 list_for_each_entry_safe(fence, fnext, &fctx->pending, head) {
131 if (fctx->read(chan) < fence->sequence)
134 nouveau_fence_signal(fence);
135 nouveau_fence_unref(&fence);
137 spin_unlock(&fctx->lock);
141 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
143 struct nouveau_fence_chan *fctx = chan->fence;
146 fence->channel = chan;
147 fence->timeout = jiffies + (15 * HZ);
148 fence->sequence = ++fctx->sequence;
150 ret = fctx->emit(fence);
152 kref_get(&fence->kref);
153 spin_lock(&fctx->lock);
154 list_add_tail(&fence->head, &fctx->pending);
155 spin_unlock(&fctx->lock);
162 nouveau_fence_done(struct nouveau_fence *fence)
165 nouveau_fence_update(fence->channel);
166 return !fence->channel;
169 struct nouveau_fence_wait {
170 struct nouveau_fence_priv *priv;
171 struct nvif_notify notify;
175 nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
177 struct nouveau_fence_wait *wait =
178 container_of(notify, typeof(*wait), notify);
179 wake_up_all(&wait->priv->waiting);
180 return NVIF_NOTIFY_KEEP;
184 nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr)
187 struct nouveau_channel *chan = fence->channel;
188 struct nouveau_fence_priv *priv = chan->drm->fence;
189 struct nouveau_fence_wait wait = { .priv = priv };
192 ret = nvif_notify_init(chan->object, NULL,
193 nouveau_fence_wait_uevent_handler, false,
194 G82_CHANNEL_DMA_V0_NTFY_UEVENT,
195 &(struct nvif_notify_uevent_req) {
197 sizeof(struct nvif_notify_uevent_req),
198 sizeof(struct nvif_notify_uevent_rep),
203 nvif_notify_get(&wait.notify);
205 if (fence->timeout) {
206 unsigned long timeout = fence->timeout - jiffies;
208 if (time_before(jiffies, fence->timeout)) {
210 ret = wait_event_interruptible_timeout(
212 nouveau_fence_done(fence),
215 ret = wait_event_timeout(priv->waiting,
216 nouveau_fence_done(fence),
222 fence->timeout = jiffies + ret;
223 if (time_after_eq(jiffies, fence->timeout))
228 ret = wait_event_interruptible(priv->waiting,
229 nouveau_fence_done(fence));
231 wait_event(priv->waiting, nouveau_fence_done(fence));
235 nvif_notify_fini(&wait.notify);
236 if (unlikely(ret < 0))
243 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
245 struct nouveau_channel *chan = fence->channel;
246 struct nouveau_fence_priv *priv = chan ? chan->drm->fence : NULL;
247 unsigned long sleep_time = NSEC_PER_MSEC / 1000;
251 while (priv && priv->uevent && lazy && !nouveau_fence_done(fence)) {
252 ret = nouveau_fence_wait_uevent(fence, intr);
257 while (!nouveau_fence_done(fence)) {
258 if (fence->timeout && time_after_eq(jiffies, fence->timeout)) {
263 __set_current_state(intr ? TASK_INTERRUPTIBLE :
264 TASK_UNINTERRUPTIBLE);
266 t = ktime_set(0, sleep_time);
267 schedule_hrtimeout(&t, HRTIMER_MODE_REL);
269 if (sleep_time > NSEC_PER_MSEC)
270 sleep_time = NSEC_PER_MSEC;
273 if (intr && signal_pending(current)) {
279 __set_current_state(TASK_RUNNING);
284 nouveau_fence_sync(struct nouveau_fence *fence, struct nouveau_channel *chan)
286 struct nouveau_fence_chan *fctx = chan->fence;
287 struct nouveau_channel *prev;
290 prev = fence ? fence->channel : NULL;
292 if (unlikely(prev != chan && !nouveau_fence_done(fence))) {
293 ret = fctx->sync(fence, prev, chan);
295 ret = nouveau_fence_wait(fence, true, false);
303 nouveau_fence_del(struct kref *kref)
305 struct nouveau_fence *fence = container_of(kref, typeof(*fence), kref);
310 nouveau_fence_unref(struct nouveau_fence **pfence)
313 kref_put(&(*pfence)->kref, nouveau_fence_del);
317 struct nouveau_fence *
318 nouveau_fence_ref(struct nouveau_fence *fence)
321 kref_get(&fence->kref);
326 nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
327 struct nouveau_fence **pfence)
329 struct nouveau_fence *fence;
332 if (unlikely(!chan->fence))
335 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
339 INIT_LIST_HEAD(&fence->work);
340 fence->sysmem = sysmem;
341 kref_init(&fence->kref);
343 ret = nouveau_fence_emit(fence, chan);
345 nouveau_fence_unref(&fence);