arch/x86/kernel/vsyscall_64.c

   1 /*
   2  *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
   3  *  Copyright 2003 Andi Kleen, SuSE Labs.
   4  *
   5  *  Thanks to hpa@transmeta.com for some useful hint.
   6  *  Special thanks to Ingo Molnar for his early experience with
   7  *  a different vsyscall implementation for Linux/IA32 and for the name.
   8  *
   9  *  vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
  10  *  at virtual address -10Mbyte+1024bytes etc... There are at max 4
  11  *  vsyscalls. One vsyscall can reserve more than 1 slot to avoid
  12  *  jumping out of line if necessary. We cannot add more with this
  13  *  mechanism because older kernels won't return -ENOSYS.
  14  *  If we want more than four we need a vDSO.
  15  *
  16  *  Note: the concept clashes with user mode linux. If you use UML and
  17  *  want per guest time just set the kernel.vsyscall64 sysctl to 0.
  18  */
  19
  20 #include <linux/time.h>
  21 #include <linux/init.h>
  22 #include <linux/kernel.h>
  23 #include <linux/timer.h>
  24 #include <linux/seqlock.h>
  25 #include <linux/jiffies.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/clocksource.h>
  28 #include <linux/getcpu.h>
  29 #include <linux/cpu.h>
  30 #include <linux/smp.h>
  31 #include <linux/notifier.h>
  32
  33 #include <asm/vsyscall.h>
  34 #include <asm/pgtable.h>
  35 #include <asm/page.h>
  36 #include <asm/unistd.h>
  37 #include <asm/fixmap.h>
  38 #include <asm/errno.h>
  39 #include <asm/io.h>
  40 #include <asm/segment.h>
  41 #include <asm/desc.h>
  42 #include <asm/topology.h>
  43 #include <asm/vgtod.h>
  44
  45 #define __vsyscall(nr) \
  46                 __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
  47 #define __syscall_clobber "r11","cx","memory"
  48
  49 /*
  50  * vsyscall_gtod_data contains data that is :
  51  * - readonly from vsyscalls
  52  * - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
  53  * Try to keep this structure as small as possible to avoid cache line ping pongs
  54  */
  55 int __vgetcpu_mode __section_vgetcpu_mode;
  56
  57 struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
  58 {
  59         .lock = SEQLOCK_UNLOCKED,
  60         .sysctl_enabled = 1,
  61 };
  62
  63 void update_vsyscall_tz(void)
  64 {
  65         unsigned long flags;
  66
  67         write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
  68         /* sys_tz has changed */
  69         vsyscall_gtod_data.sys_tz = sys_tz;
  70         write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
  71 }
  72
  73 void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
  74 {
  75         unsigned long flags;
  76
  77         write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
  78         /* copy vsyscall data */
  79         vsyscall_gtod_data.clock.vread = clock->vread;
  80         vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
  81         vsyscall_gtod_data.clock.mask = clock->mask;
  82         vsyscall_gtod_data.clock.mult = clock->mult;
  83         vsyscall_gtod_data.clock.shift = clock->shift;
  84         vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
  85         vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
  86         vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
  87         write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
  88 }
  89
  90 /* RED-PEN may want to readd seq locking, but then the variable should be
  91  * write-once.
  92  */
  93 static __always_inline void do_get_tz(struct timezone * tz)
  94 {
  95         *tz = __vsyscall_gtod_data.sys_tz;
  96 }
  97
  98 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
  99 {
 100         int ret;
 101         asm volatile("syscall"
 102                 : "=a" (ret)
 103                 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
 104                 : __syscall_clobber );
 105         return ret;
 106 }
 107
 108 static __always_inline long time_syscall(long *t)
 109 {
 110         long secs;
 111         asm volatile("syscall"
 112                 : "=a" (secs)
 113                 : "0" (__NR_time),"D" (t) : __syscall_clobber);
 114         return secs;
 115 }
 116
 117 static __always_inline void do_vgettimeofday(struct timeval * tv)
 118 {
 119         cycle_t now, base, mask, cycle_delta;
 120         unsigned seq;
 121         unsigned long mult, shift, nsec;
 122         cycle_t (*vread)(void);
 123         do {
 124                 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
 125
 126                 vread = __vsyscall_gtod_data.clock.vread;
 127                 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
 128                         gettimeofday(tv,NULL);
 129                         return;
 130                 }
 131
 132                 /*
 133                  * Surround the RDTSC by barriers, to make sure it's not
 134                  * speculated to outside the seqlock critical section and
 135                  * does not cause time warps:
 136                  */
 137                 rdtsc_barrier();
 138                 now = vread();
 139                 rdtsc_barrier();
 140
 141                 base = __vsyscall_gtod_data.clock.cycle_last;
 142                 mask = __vsyscall_gtod_data.clock.mask;
 143                 mult = __vsyscall_gtod_data.clock.mult;
 144                 shift = __vsyscall_gtod_data.clock.shift;
 145
 146                 tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
 147                 nsec = __vsyscall_gtod_data.wall_time_nsec;
 148         } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
 149
 150         /* calculate interval: */
 151         cycle_delta = (now - base) & mask;
 152         /* convert to nsecs: */
 153         nsec += (cycle_delta * mult) >> shift;
 154
 155         while (nsec >= NSEC_PER_SEC) {
 156                 tv->tv_sec += 1;
 157                 nsec -= NSEC_PER_SEC;
 158         }
 159         tv->tv_usec = nsec / NSEC_PER_USEC;
 160 }
 161
 162 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
 163 {
 164         if (tv)
 165                 do_vgettimeofday(tv);
 166         if (tz)
 167                 do_get_tz(tz);
 168         return 0;
 169 }
 170
 171 /* This will break when the xtime seconds get inaccurate, but that is
 172  * unlikely */
 173 time_t __vsyscall(1) vtime(time_t *t)
 174 {
 175         struct timeval tv;
 176         time_t result;
 177         if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
 178                 return time_syscall(t);
 179
 180         vgettimeofday(&tv, NULL);
 181         result = tv.tv_sec;
 182         if (t)
 183                 *t = result;
 184         return result;
 185 }
 186
 187 /* Fast way to get current CPU and node.
 188    This helps to do per node and per CPU caches in user space.
 189    The result is not guaranteed without CPU affinity, but usually
 190    works out because the scheduler tries to keep a thread on the same
 191    CPU.
 192
 193    tcache must point to a two element sized long array.
 194    All arguments can be NULL. */
 195 long __vsyscall(2)
 196 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 197 {
 198         unsigned int p;
 199         unsigned long j = 0;
 200
 201         /* Fast cache - only recompute value once per jiffies and avoid
 202            relatively costly rdtscp/cpuid otherwise.
 203            This works because the scheduler usually keeps the process
 204            on the same CPU and this syscall doesn't guarantee its
 205            results anyways.
 206            We do this here because otherwise user space would do it on
 207            its own in a likely inferior way (no access to jiffies).
 208            If you don't like it pass NULL. */
 209         if (tcache && tcache->blob[0] == (j = __jiffies)) {
 210                 p = tcache->blob[1];
 211         } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
 212                 /* Load per CPU data from RDTSCP */
 213                 native_read_tscp(&p);
 214         } else {
 215                 /* Load per CPU data from GDT */
 216                 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
 217         }
 218         if (tcache) {
 219                 tcache->blob[0] = j;
 220                 tcache->blob[1] = p;
 221         }
 222         if (cpu)
 223                 *cpu = p & 0xfff;
 224         if (node)
 225                 *node = p >> 12;
 226         return 0;
 227 }
 228
 229 static long __vsyscall(3) venosys_1(void)
 230 {
 231         return -ENOSYS;
 232 }
 233
 234 #ifdef CONFIG_SYSCTL
 235
 236 static int
 237 vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
 238                        void __user *buffer, size_t *lenp, loff_t *ppos)
 239 {
 240         return proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
 241 }
 242
 243 static ctl_table kernel_table2[] = {
 244         { .procname = "vsyscall64",
 245           .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
 246           .mode = 0644,
 247           .proc_handler = vsyscall_sysctl_change },
 248         {}
 249 };
 250
 251 static ctl_table kernel_root_table2[] = {
 252         { .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
 253           .child = kernel_table2 },
 254         {}
 255 };
 256 #endif
 257
 258 /* Assume __initcall executes before all user space. Hopefully kmod
 259    doesn't violate that. We'll find out if it does. */
 260 static void __cpuinit vsyscall_set_cpu(int cpu)
 261 {
 262         unsigned long d;
 263         unsigned long node = 0;
 264 #ifdef CONFIG_NUMA
 265         node = cpu_to_node(cpu);
 266 #endif
 267         if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
 268                 write_rdtscp_aux((node << 12) | cpu);
 269
 270         /* Store cpu number in limit so that it can be loaded quickly
 271            in user space in vgetcpu.
 272            12 bits for the CPU and 8 bits for the node. */
 273         d = 0x0f40000000000ULL;
 274         d |= cpu;
 275         d |= (node & 0xf) << 12;
 276         d |= (node >> 4) << 48;
 277         write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
 278 }
 279
 280 static void __cpuinit cpu_vsyscall_init(void *arg)
 281 {
 282         /* preemption should be already off */
 283         vsyscall_set_cpu(raw_smp_processor_id());
 284 }
 285
 286 static int __cpuinit
 287 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
 288 {
 289         long cpu = (long)arg;
 290         if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
 291                 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
 292         return NOTIFY_DONE;
 293 }
 294
 295 void __init map_vsyscall(void)
 296 {
 297         extern char __vsyscall_0;
 298         unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
 299
 300         /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
 301         __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
 302 }
 303
 304 static int __init vsyscall_init(void)
 305 {
 306         BUG_ON(((unsigned long) &vgettimeofday !=
 307                         VSYSCALL_ADDR(__NR_vgettimeofday)));
 308         BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
 309         BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
 310         BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
 311 #ifdef CONFIG_SYSCTL
 312         register_sysctl_table(kernel_root_table2);
 313 #endif
 314         on_each_cpu(cpu_vsyscall_init, NULL, 1);
 315         hotcpu_notifier(cpu_vsyscall_notifier, 0);
 316         return 0;
 317 }
 318
 319 __initcall(vsyscall_init);