2 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
3 * Copyright 2003 Andi Kleen, SuSE Labs.
5 * Thanks to hpa@transmeta.com for some useful hint.
6 * Special thanks to Ingo Molnar for his early experience with
7 * a different vsyscall implementation for Linux/IA32 and for the name.
9 * vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
10 * at virtual address -10Mbyte+1024bytes etc... There are at max 4
11 * vsyscalls. One vsyscall can reserve more than 1 slot to avoid
12 * jumping out of line if necessary. We cannot add more with this
13 * mechanism because older kernels won't return -ENOSYS.
14 * If we want more than four we need a vDSO.
16 * Note: the concept clashes with user mode linux. If you use UML and
17 * want per guest time just set the kernel.vsyscall64 sysctl to 0.
20 #include <linux/time.h>
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/timer.h>
24 #include <linux/seqlock.h>
25 #include <linux/jiffies.h>
26 #include <linux/sysctl.h>
27 #include <linux/clocksource.h>
28 #include <linux/getcpu.h>
29 #include <linux/cpu.h>
30 #include <linux/smp.h>
31 #include <linux/notifier.h>
33 #include <asm/vsyscall.h>
34 #include <asm/pgtable.h>
36 #include <asm/unistd.h>
37 #include <asm/fixmap.h>
38 #include <asm/errno.h>
40 #include <asm/segment.h>
42 #include <asm/topology.h>
43 #include <asm/vgtod.h>
45 #define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
46 #define __syscall_clobber "r11","rcx","memory"
47 #define __pa_vsymbol(x) \
49 extern char __vsyscall_0; \
50 asm("" : "=r" (v) : "0" (x)); \
51 ((v - VSYSCALL_FIRST_PAGE) + __pa_symbol(&__vsyscall_0)); })
54 * vsyscall_gtod_data contains data that is :
55 * - readonly from vsyscalls
56 * - writen by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
57 * Try to keep this structure as small as possible to avoid cache line ping pongs
59 int __vgetcpu_mode __section_vgetcpu_mode;
61 struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
63 .lock = SEQLOCK_UNLOCKED,
67 void update_vsyscall_tz(void)
71 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
72 /* sys_tz has changed */
73 vsyscall_gtod_data.sys_tz = sys_tz;
74 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
77 void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
81 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
82 /* copy vsyscall data */
83 vsyscall_gtod_data.clock.vread = clock->vread;
84 vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
85 vsyscall_gtod_data.clock.mask = clock->mask;
86 vsyscall_gtod_data.clock.mult = clock->mult;
87 vsyscall_gtod_data.clock.shift = clock->shift;
88 vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
89 vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
90 vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
91 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
94 /* RED-PEN may want to readd seq locking, but then the variable should be
97 static __always_inline void do_get_tz(struct timezone * tz)
99 *tz = __vsyscall_gtod_data.sys_tz;
102 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
105 asm volatile("vsysc2: syscall"
107 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
108 : __syscall_clobber );
112 static __always_inline long time_syscall(long *t)
115 asm volatile("vsysc1: syscall"
117 : "0" (__NR_time),"D" (t) : __syscall_clobber);
121 static __always_inline void do_vgettimeofday(struct timeval * tv)
123 cycle_t now, base, mask, cycle_delta;
125 unsigned long mult, shift, nsec;
126 cycle_t (*vread)(void);
128 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
130 vread = __vsyscall_gtod_data.clock.vread;
131 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
132 gettimeofday(tv,NULL);
136 base = __vsyscall_gtod_data.clock.cycle_last;
137 mask = __vsyscall_gtod_data.clock.mask;
138 mult = __vsyscall_gtod_data.clock.mult;
139 shift = __vsyscall_gtod_data.clock.shift;
141 tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
142 nsec = __vsyscall_gtod_data.wall_time_nsec;
143 } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
145 /* calculate interval: */
146 cycle_delta = (now - base) & mask;
147 /* convert to nsecs: */
148 nsec += (cycle_delta * mult) >> shift;
150 while (nsec >= NSEC_PER_SEC) {
152 nsec -= NSEC_PER_SEC;
154 tv->tv_usec = nsec / NSEC_PER_USEC;
157 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
160 do_vgettimeofday(tv);
166 /* This will break when the xtime seconds get inaccurate, but that is
168 time_t __vsyscall(1) vtime(time_t *t)
172 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
173 return time_syscall(t);
175 vgettimeofday(&tv, 0);
182 /* Fast way to get current CPU and node.
183 This helps to do per node and per CPU caches in user space.
184 The result is not guaranteed without CPU affinity, but usually
185 works out because the scheduler tries to keep a thread on the same
188 tcache must point to a two element sized long array.
189 All arguments can be NULL. */
191 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
193 unsigned int dummy, p;
196 /* Fast cache - only recompute value once per jiffies and avoid
197 relatively costly rdtscp/cpuid otherwise.
198 This works because the scheduler usually keeps the process
199 on the same CPU and this syscall doesn't guarantee its
201 We do this here because otherwise user space would do it on
202 its own in a likely inferior way (no access to jiffies).
203 If you don't like it pass NULL. */
204 if (tcache && tcache->blob[0] == (j = __jiffies)) {
206 } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
207 /* Load per CPU data from RDTSCP */
208 rdtscp(dummy, dummy, p);
210 /* Load per CPU data from GDT */
211 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
224 long __vsyscall(3) venosys_1(void)
231 #define SYSCALL 0x050f
235 * NOP out syscall in vsyscall page when not needed.
237 static int vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
238 void __user *buffer, size_t *lenp, loff_t *ppos)
240 extern u16 vsysc1, vsysc2;
243 int ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
246 /* gcc has some trouble with __va(__pa()), so just do it this
248 map1 = ioremap(__pa_vsymbol(&vsysc1), 2);
251 map2 = ioremap(__pa_vsymbol(&vsysc2), 2);
256 if (!vsyscall_gtod_data.sysctl_enabled) {
257 writew(SYSCALL, map1);
258 writew(SYSCALL, map2);
269 static int vsyscall_sysctl_nostrat(ctl_table *t, int __user *name, int nlen,
270 void __user *oldval, size_t __user *oldlenp,
271 void __user *newval, size_t newlen)
276 static ctl_table kernel_table2[] = {
277 { .ctl_name = 99, .procname = "vsyscall64",
278 .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
280 .strategy = vsyscall_sysctl_nostrat,
281 .proc_handler = vsyscall_sysctl_change },
285 static ctl_table kernel_root_table2[] = {
286 { .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
287 .child = kernel_table2 },
293 /* Assume __initcall executes before all user space. Hopefully kmod
294 doesn't violate that. We'll find out if it does. */
295 static void __cpuinit vsyscall_set_cpu(int cpu)
298 unsigned long node = 0;
300 node = cpu_to_node(cpu);
302 if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
303 write_rdtscp_aux((node << 12) | cpu);
305 /* Store cpu number in limit so that it can be loaded quickly
306 in user space in vgetcpu.
307 12 bits for the CPU and 8 bits for the node. */
308 d = (unsigned long *)(cpu_gdt(cpu) + GDT_ENTRY_PER_CPU);
309 *d = 0x0f40000000000ULL;
311 *d |= (node & 0xf) << 12;
312 *d |= (node >> 4) << 48;
315 static void __cpuinit cpu_vsyscall_init(void *arg)
317 /* preemption should be already off */
318 vsyscall_set_cpu(raw_smp_processor_id());
322 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
324 long cpu = (long)arg;
325 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
326 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
330 static void __init map_vsyscall(void)
332 extern char __vsyscall_0;
333 unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
335 /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
336 __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
339 static int __init vsyscall_init(void)
341 BUG_ON(((unsigned long) &vgettimeofday !=
342 VSYSCALL_ADDR(__NR_vgettimeofday)));
343 BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
344 BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
345 BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
348 register_sysctl_table(kernel_root_table2);
350 on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
351 hotcpu_notifier(cpu_vsyscall_notifier, 0);
355 __initcall(vsyscall_init);