1 /* sched.c - SPU scheduler.
3 * Copyright (C) IBM 2005
4 * Author: Mark Nutter <mnutter@us.ibm.com>
6 * 2006-03-31 NUMA domains added.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25 #include <linux/module.h>
26 #include <linux/errno.h>
27 #include <linux/sched.h>
28 #include <linux/kernel.h>
30 #include <linux/completion.h>
31 #include <linux/vmalloc.h>
32 #include <linux/smp.h>
33 #include <linux/smp_lock.h>
34 #include <linux/stddef.h>
35 #include <linux/unistd.h>
36 #include <linux/numa.h>
37 #include <linux/mutex.h>
38 #include <linux/notifier.h>
41 #include <asm/mmu_context.h>
43 #include <asm/spu_csa.h>
44 #include <asm/spu_priv1.h>
47 #define SPU_MIN_TIMESLICE (100 * HZ / 1000)
49 #define SPU_BITMAP_SIZE (((MAX_PRIO+BITS_PER_LONG)/BITS_PER_LONG)+1)
50 struct spu_prio_array {
51 unsigned long bitmap[SPU_BITMAP_SIZE];
52 wait_queue_head_t waitq[MAX_PRIO];
53 struct list_head active_list[MAX_NUMNODES];
54 struct mutex active_mutex[MAX_NUMNODES];
57 static struct spu_prio_array *spu_prio;
59 static inline int node_allowed(int node)
63 if (!nr_cpus_node(node))
65 mask = node_to_cpumask(node);
66 if (!cpus_intersects(mask, current->cpus_allowed))
71 static inline void mm_needs_global_tlbie(struct mm_struct *mm)
73 int nr = (NR_CPUS > 1) ? NR_CPUS : NR_CPUS + 1;
75 /* Global TLBIE broadcast required with SPEs. */
76 __cpus_setall(&mm->cpu_vm_mask, nr);
79 static BLOCKING_NOTIFIER_HEAD(spu_switch_notifier);
81 static void spu_switch_notify(struct spu *spu, struct spu_context *ctx)
83 blocking_notifier_call_chain(&spu_switch_notifier,
84 ctx ? ctx->object_id : 0, spu);
87 int spu_switch_event_register(struct notifier_block * n)
89 return blocking_notifier_chain_register(&spu_switch_notifier, n);
92 int spu_switch_event_unregister(struct notifier_block * n)
94 return blocking_notifier_chain_unregister(&spu_switch_notifier, n);
98 static inline void bind_context(struct spu *spu, struct spu_context *ctx)
100 pr_debug("%s: pid=%d SPU=%d NODE=%d\n", __FUNCTION__, current->pid,
101 spu->number, spu->node);
105 ctx->ops = &spu_hw_ops;
106 spu->pid = current->pid;
107 spu->prio = current->prio;
108 spu->mm = ctx->owner;
109 mm_needs_global_tlbie(spu->mm);
110 spu->ibox_callback = spufs_ibox_callback;
111 spu->wbox_callback = spufs_wbox_callback;
112 spu->stop_callback = spufs_stop_callback;
113 spu->mfc_callback = spufs_mfc_callback;
114 spu->dma_callback = spufs_dma_callback;
116 spu_unmap_mappings(ctx);
117 spu_restore(&ctx->csa, spu);
118 spu->timestamp = jiffies;
119 spu_cpu_affinity_set(spu, raw_smp_processor_id());
120 spu_switch_notify(spu, ctx);
122 ctx->state = SPU_STATE_RUNNABLE;
125 static inline void unbind_context(struct spu *spu, struct spu_context *ctx)
127 pr_debug("%s: unbind pid=%d SPU=%d NODE=%d\n", __FUNCTION__,
128 spu->pid, spu->number, spu->node);
129 spu_switch_notify(spu, NULL);
130 spu_unmap_mappings(ctx);
131 spu_save(&ctx->csa, spu);
132 spu->timestamp = jiffies;
133 ctx->state = SPU_STATE_SAVED;
134 spu->ibox_callback = NULL;
135 spu->wbox_callback = NULL;
136 spu->stop_callback = NULL;
137 spu->mfc_callback = NULL;
138 spu->dma_callback = NULL;
141 spu->prio = MAX_PRIO;
142 ctx->ops = &spu_backing_ops;
148 static inline void spu_add_wq(wait_queue_head_t * wq, wait_queue_t * wait,
151 prepare_to_wait_exclusive(wq, wait, TASK_INTERRUPTIBLE);
152 set_bit(prio, spu_prio->bitmap);
155 static inline void spu_del_wq(wait_queue_head_t * wq, wait_queue_t * wait,
160 __set_current_state(TASK_RUNNING);
162 spin_lock_irqsave(&wq->lock, flags);
164 remove_wait_queue_locked(wq, wait);
165 if (list_empty(&wq->task_list))
166 clear_bit(prio, spu_prio->bitmap);
168 spin_unlock_irqrestore(&wq->lock, flags);
171 static void spu_prio_wait(struct spu_context *ctx, u64 flags)
173 int prio = current->prio;
174 wait_queue_head_t *wq = &spu_prio->waitq[prio];
180 spu_add_wq(wq, &wait, prio);
182 if (!signal_pending(current)) {
183 up_write(&ctx->state_sema);
184 pr_debug("%s: pid=%d prio=%d\n", __FUNCTION__,
185 current->pid, current->prio);
187 down_write(&ctx->state_sema);
190 spu_del_wq(wq, &wait, prio);
193 static void spu_prio_wakeup(void)
195 int best = sched_find_first_bit(spu_prio->bitmap);
196 if (best < MAX_PRIO) {
197 wait_queue_head_t *wq = &spu_prio->waitq[best];
198 wake_up_interruptible_nr(wq, 1);
202 static int get_active_spu(struct spu *spu)
204 int node = spu->node;
208 mutex_lock(&spu_prio->active_mutex[node]);
209 list_for_each_entry(tmp, &spu_prio->active_list[node], list) {
211 list_del_init(&spu->list);
216 mutex_unlock(&spu_prio->active_mutex[node]);
220 static void put_active_spu(struct spu *spu)
222 int node = spu->node;
224 mutex_lock(&spu_prio->active_mutex[node]);
225 list_add_tail(&spu->list, &spu_prio->active_list[node]);
226 mutex_unlock(&spu_prio->active_mutex[node]);
229 static struct spu *spu_get_idle(struct spu_context *ctx, u64 flags)
231 struct spu *spu = NULL;
232 int node = cpu_to_node(raw_smp_processor_id());
235 for (n = 0; n < MAX_NUMNODES; n++, node++) {
236 node = (node < MAX_NUMNODES) ? node : 0;
237 if (!node_allowed(node))
239 spu = spu_alloc_node(node);
246 static inline struct spu *spu_get(struct spu_context *ctx, u64 flags)
248 /* Future: spu_get_idle() if possible,
249 * otherwise try to preempt an active
252 return spu_get_idle(ctx, flags);
255 /* The three externally callable interfaces
256 * for the scheduler begin here.
258 * spu_activate - bind a context to SPU, waiting as needed.
259 * spu_deactivate - unbind a context from its SPU.
260 * spu_yield - yield an SPU if others are waiting.
263 int spu_activate(struct spu_context *ctx, u64 flags)
271 spu = spu_get(ctx, flags);
273 if (ctx->spu != NULL) {
278 bind_context(spu, ctx);
282 spu_prio_wait(ctx, flags);
283 if (signal_pending(current)) {
292 void spu_deactivate(struct spu_context *ctx)
300 needs_idle = get_active_spu(spu);
301 unbind_context(spu, ctx);
308 void spu_yield(struct spu_context *ctx)
313 if (down_write_trylock(&ctx->state_sema)) {
314 if ((spu = ctx->spu) != NULL) {
315 int best = sched_find_first_bit(spu_prio->bitmap);
316 if (best < MAX_PRIO) {
317 pr_debug("%s: yielding SPU %d NODE %d\n",
318 __FUNCTION__, spu->number, spu->node);
322 spu->prio = MAX_PRIO;
325 up_write(&ctx->state_sema);
327 if (unlikely(need_yield))
331 int __init spu_sched_init(void)
335 spu_prio = kzalloc(sizeof(struct spu_prio_array), GFP_KERNEL);
337 printk(KERN_WARNING "%s: Unable to allocate priority queue.\n",
341 for (i = 0; i < MAX_PRIO; i++) {
342 init_waitqueue_head(&spu_prio->waitq[i]);
343 __clear_bit(i, spu_prio->bitmap);
345 __set_bit(MAX_PRIO, spu_prio->bitmap);
346 for (i = 0; i < MAX_NUMNODES; i++) {
347 mutex_init(&spu_prio->active_mutex[i]);
348 INIT_LIST_HEAD(&spu_prio->active_list[i]);
353 void __exit spu_sched_exit(void)
355 struct spu *spu, *tmp;
358 for (node = 0; node < MAX_NUMNODES; node++) {
359 mutex_lock(&spu_prio->active_mutex[node]);
360 list_for_each_entry_safe(spu, tmp, &spu_prio->active_list[node],
362 list_del_init(&spu->list);
365 mutex_unlock(&spu_prio->active_mutex[node]);