arch/x86/vdso/vclock_gettime.c

   1 /*
   2  * Copyright 2006 Andi Kleen, SUSE Labs.
   3  * Subject to the GNU Public License, v.2
   4  *
   5  * Fast user context implementation of clock_gettime, gettimeofday, and time.
   6  *
   7  * 32 Bit compat layer by Stefani Seibold <stefani@seibold.net>
   8  *  sponsored by Rohde & Schwarz GmbH & Co. KG Munich/Germany
   9  *
  10  * The code should have no internal unresolved relocations.
  11  * Check with readelf after changing.
  12  */
  13
  14 #include <uapi/linux/time.h>
  15 #include <asm/vgtod.h>
  16 #include <asm/hpet.h>
  17 #include <asm/vvar.h>
  18 #include <asm/unistd.h>
  19 #include <asm/msr.h>
  20 #include <linux/math64.h>
  21 #include <linux/time.h>
  22
  23 #define gtod (&VVAR(vsyscall_gtod_data))
  24
  25 extern int __vdso_clock_gettime(clockid_t clock, struct timespec *ts);
  26 extern int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz);
  27 extern time_t __vdso_time(time_t *t);
  28
  29 #ifdef CONFIG_HPET_TIMER
  30 extern u8 hpet_page
  31         __attribute__((visibility("hidden")));
  32
  33 static notrace cycle_t vread_hpet(void)
  34 {
  35         return *(const volatile u32 *)(&hpet_page + HPET_COUNTER);
  36 }
  37 #endif
  38
  39 #ifndef BUILD_VDSO32
  40
  41 #include <linux/kernel.h>
  42 #include <asm/vsyscall.h>
  43 #include <asm/fixmap.h>
  44 #include <asm/pvclock.h>
  45
  46 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
  47 {
  48         long ret;
  49         asm("syscall" : "=a" (ret) :
  50             "0" (__NR_clock_gettime), "D" (clock), "S" (ts) : "memory");
  51         return ret;
  52 }
  53
  54 notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
  55 {
  56         long ret;
  57
  58         asm("syscall" : "=a" (ret) :
  59             "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
  60         return ret;
  61 }
  62
  63 #ifdef CONFIG_PARAVIRT_CLOCK
  64
  65 static notrace const struct pvclock_vsyscall_time_info *get_pvti(int cpu)
  66 {
  67         const struct pvclock_vsyscall_time_info *pvti_base;
  68         int idx = cpu / (PAGE_SIZE/PVTI_SIZE);
  69         int offset = cpu % (PAGE_SIZE/PVTI_SIZE);
  70
  71         BUG_ON(PVCLOCK_FIXMAP_BEGIN + idx > PVCLOCK_FIXMAP_END);
  72
  73         pvti_base = (struct pvclock_vsyscall_time_info *)
  74                     __fix_to_virt(PVCLOCK_FIXMAP_BEGIN+idx);
  75
  76         return &pvti_base[offset];
  77 }
  78
  79 static notrace cycle_t vread_pvclock(int *mode)
  80 {
  81         const struct pvclock_vsyscall_time_info *pvti;
  82         cycle_t ret;
  83         u64 last;
  84         u32 version;
  85         u32 migrate_count;
  86         u8 flags;
  87         unsigned cpu, cpu1;
  88
  89
  90         /*
  91          * When looping to get a consistent (time-info, tsc) pair, we
  92          * also need to deal with the possibility we can switch vcpus,
  93          * so make sure we always re-fetch time-info for the current vcpu.
  94          */
  95         do {
  96                 cpu = __getcpu() & VGETCPU_CPU_MASK;
  97                 /* TODO: We can put vcpu id into higher bits of pvti.version.
  98                  * This will save a couple of cycles by getting rid of
  99                  * __getcpu() calls (Gleb).
 100                  */
 101
 102                 /* Make sure migrate_count will change if we leave the VCPU. */
 103                 do {
 104                         pvti = get_pvti(cpu);
 105                         migrate_count = pvti->migrate_count;
 106
 107                         cpu1 = cpu;
 108                         cpu = __getcpu() & VGETCPU_CPU_MASK;
 109                 } while (unlikely(cpu != cpu1));
 110
 111                 version = __pvclock_read_cycles(&pvti->pvti, &ret, &flags);
 112
 113                 /*
 114                  * Test we're still on the cpu as well as the version.
 115                  * - We must read TSC of pvti's VCPU.
 116                  * - KVM doesn't follow the versioning protocol, so data could
 117                  *   change before version if we left the VCPU.
 118                  */
 119                 smp_rmb();
 120         } while (unlikely((pvti->pvti.version & 1) ||
 121                           pvti->pvti.version != version ||
 122                           pvti->migrate_count != migrate_count));
 123
 124         if (unlikely(!(flags & PVCLOCK_TSC_STABLE_BIT)))
 125                 *mode = VCLOCK_NONE;
 126
 127         /* refer to tsc.c read_tsc() comment for rationale */
 128         last = gtod->cycle_last;
 129
 130         if (likely(ret >= last))
 131                 return ret;
 132
 133         return last;
 134 }
 135 #endif
 136
 137 #else
 138
 139 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
 140 {
 141         long ret;
 142
 143         asm(
 144                 "mov %%ebx, %%edx \n"
 145                 "mov %2, %%ebx \n"
 146                 "call __kernel_vsyscall \n"
 147                 "mov %%edx, %%ebx \n"
 148                 : "=a" (ret)
 149                 : "0" (__NR_clock_gettime), "g" (clock), "c" (ts)
 150                 : "memory", "edx");
 151         return ret;
 152 }
 153
 154 notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
 155 {
 156         long ret;
 157
 158         asm(
 159                 "mov %%ebx, %%edx \n"
 160                 "mov %2, %%ebx \n"
 161                 "call __kernel_vsyscall \n"
 162                 "mov %%edx, %%ebx \n"
 163                 : "=a" (ret)
 164                 : "0" (__NR_gettimeofday), "g" (tv), "c" (tz)
 165                 : "memory", "edx");
 166         return ret;
 167 }
 168
 169 #ifdef CONFIG_PARAVIRT_CLOCK
 170
 171 static notrace cycle_t vread_pvclock(int *mode)
 172 {
 173         *mode = VCLOCK_NONE;
 174         return 0;
 175 }
 176 #endif
 177
 178 #endif
 179
 180 notrace static cycle_t vread_tsc(void)
 181 {
 182         cycle_t ret;
 183         u64 last;
 184
 185         /*
 186          * Empirically, a fence (of type that depends on the CPU)
 187          * before rdtsc is enough to ensure that rdtsc is ordered
 188          * with respect to loads.  The various CPU manuals are unclear
 189          * as to whether rdtsc can be reordered with later loads,
 190          * but no one has ever seen it happen.
 191          */
 192         rdtsc_barrier();
 193         ret = (cycle_t)__native_read_tsc();
 194
 195         last = gtod->cycle_last;
 196
 197         if (likely(ret >= last))
 198                 return ret;
 199
 200         /*
 201          * GCC likes to generate cmov here, but this branch is extremely
 202          * predictable (it's just a funciton of time and the likely is
 203          * very likely) and there's a data dependence, so force GCC
 204          * to generate a branch instead.  I don't barrier() because
 205          * we don't actually need a barrier, and if this function
 206          * ever gets inlined it will generate worse code.
 207          */
 208         asm volatile ("");
 209         return last;
 210 }
 211
 212 notrace static inline u64 vgetsns(int *mode)
 213 {
 214         u64 v;
 215         cycles_t cycles;
 216
 217         if (gtod->vclock_mode == VCLOCK_TSC)
 218                 cycles = vread_tsc();
 219 #ifdef CONFIG_HPET_TIMER
 220         else if (gtod->vclock_mode == VCLOCK_HPET)
 221                 cycles = vread_hpet();
 222 #endif
 223 #ifdef CONFIG_PARAVIRT_CLOCK
 224         else if (gtod->vclock_mode == VCLOCK_PVCLOCK)
 225                 cycles = vread_pvclock(mode);
 226 #endif
 227         else
 228                 return 0;
 229         v = (cycles - gtod->cycle_last) & gtod->mask;
 230         return v * gtod->mult;
 231 }
 232
 233 /* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
 234 notrace static int __always_inline do_realtime(struct timespec *ts)
 235 {
 236         unsigned long seq;
 237         u64 ns;
 238         int mode;
 239
 240         do {
 241                 seq = gtod_read_begin(gtod);
 242                 mode = gtod->vclock_mode;
 243                 ts->tv_sec = gtod->wall_time_sec;
 244                 ns = gtod->wall_time_snsec;
 245                 ns += vgetsns(&mode);
 246                 ns >>= gtod->shift;
 247         } while (unlikely(gtod_read_retry(gtod, seq)));
 248
 249         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 250         ts->tv_nsec = ns;
 251
 252         return mode;
 253 }
 254
 255 notrace static int __always_inline do_monotonic(struct timespec *ts)
 256 {
 257         unsigned long seq;
 258         u64 ns;
 259         int mode;
 260
 261         do {
 262                 seq = gtod_read_begin(gtod);
 263                 mode = gtod->vclock_mode;
 264                 ts->tv_sec = gtod->monotonic_time_sec;
 265                 ns = gtod->monotonic_time_snsec;
 266                 ns += vgetsns(&mode);
 267                 ns >>= gtod->shift;
 268         } while (unlikely(gtod_read_retry(gtod, seq)));
 269
 270         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 271         ts->tv_nsec = ns;
 272
 273         return mode;
 274 }
 275
 276 notrace static void do_realtime_coarse(struct timespec *ts)
 277 {
 278         unsigned long seq;
 279         do {
 280                 seq = gtod_read_begin(gtod);
 281                 ts->tv_sec = gtod->wall_time_coarse_sec;
 282                 ts->tv_nsec = gtod->wall_time_coarse_nsec;
 283         } while (unlikely(gtod_read_retry(gtod, seq)));
 284 }
 285
 286 notrace static void do_monotonic_coarse(struct timespec *ts)
 287 {
 288         unsigned long seq;
 289         do {
 290                 seq = gtod_read_begin(gtod);
 291                 ts->tv_sec = gtod->monotonic_time_coarse_sec;
 292                 ts->tv_nsec = gtod->monotonic_time_coarse_nsec;
 293         } while (unlikely(gtod_read_retry(gtod, seq)));
 294 }
 295
 296 notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
 297 {
 298         switch (clock) {
 299         case CLOCK_REALTIME:
 300                 if (do_realtime(ts) == VCLOCK_NONE)
 301                         goto fallback;
 302                 break;
 303         case CLOCK_MONOTONIC:
 304                 if (do_monotonic(ts) == VCLOCK_NONE)
 305                         goto fallback;
 306                 break;
 307         case CLOCK_REALTIME_COARSE:
 308                 do_realtime_coarse(ts);
 309                 break;
 310         case CLOCK_MONOTONIC_COARSE:
 311                 do_monotonic_coarse(ts);
 312                 break;
 313         default:
 314                 goto fallback;
 315         }
 316
 317         return 0;
 318 fallback:
 319         return vdso_fallback_gettime(clock, ts);
 320 }
 321 int clock_gettime(clockid_t, struct timespec *)
 322         __attribute__((weak, alias("__vdso_clock_gettime")));
 323
 324 notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
 325 {
 326         if (likely(tv != NULL)) {
 327                 if (unlikely(do_realtime((struct timespec *)tv) == VCLOCK_NONE))
 328                         return vdso_fallback_gtod(tv, tz);
 329                 tv->tv_usec /= 1000;
 330         }
 331         if (unlikely(tz != NULL)) {
 332                 tz->tz_minuteswest = gtod->tz_minuteswest;
 333                 tz->tz_dsttime = gtod->tz_dsttime;
 334         }
 335
 336         return 0;
 337 }
 338 int gettimeofday(struct timeval *, struct timezone *)
 339         __attribute__((weak, alias("__vdso_gettimeofday")));
 340
 341 /*
 342  * This will break when the xtime seconds get inaccurate, but that is
 343  * unlikely
 344  */
 345 notrace time_t __vdso_time(time_t *t)
 346 {
 347         /* This is atomic on x86 so we don't need any locks. */
 348         time_t result = ACCESS_ONCE(gtod->wall_time_sec);
 349
 350         if (t)
 351                 *t = result;
 352         return result;
 353 }
 354 int time(time_t *t)
 355         __attribute__((weak, alias("__vdso_time")));