timekeeping: Update clocksource with stop_machine
[deliverable/linux.git] / kernel / time / timekeeping.c
CommitLineData
8524070b 1/*
2 * linux/kernel/time/timekeeping.c
3 *
4 * Kernel timekeeping code and accessor functions
5 *
6 * This code was moved from linux/kernel/timer.c.
7 * Please see that file for copyright and history logs.
8 *
9 */
10
11#include <linux/module.h>
12#include <linux/interrupt.h>
13#include <linux/percpu.h>
14#include <linux/init.h>
15#include <linux/mm.h>
16#include <linux/sysdev.h>
17#include <linux/clocksource.h>
18#include <linux/jiffies.h>
19#include <linux/time.h>
20#include <linux/tick.h>
75c5158f 21#include <linux/stop_machine.h>
8524070b 22
155ec602
MS
23/* Structure holding internal timekeeping values. */
24struct timekeeper {
25 /* Current clocksource used for timekeeping. */
26 struct clocksource *clock;
23ce7211
MS
27 /* The shift value of the current clocksource. */
28 int shift;
155ec602
MS
29
30 /* Number of clock cycles in one NTP interval. */
31 cycle_t cycle_interval;
32 /* Number of clock shifted nano seconds in one NTP interval. */
33 u64 xtime_interval;
34 /* Raw nano seconds accumulated per NTP interval. */
35 u32 raw_interval;
36
37 /* Clock shifted nano seconds remainder not stored in xtime.tv_nsec. */
38 u64 xtime_nsec;
39 /* Difference between accumulated time and NTP time in ntp
40 * shifted nano seconds. */
41 s64 ntp_error;
23ce7211
MS
42 /* Shift conversion between clock shifted nano seconds and
43 * ntp shifted nano seconds. */
44 int ntp_error_shift;
0a544198
MS
45 /* NTP adjusted clock multiplier */
46 u32 mult;
155ec602
MS
47};
48
49struct timekeeper timekeeper;
50
51/**
52 * timekeeper_setup_internals - Set up internals to use clocksource clock.
53 *
54 * @clock: Pointer to clocksource.
55 *
56 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
57 * pair and interval request.
58 *
59 * Unless you're the timekeeping code, you should not be using this!
60 */
61static void timekeeper_setup_internals(struct clocksource *clock)
62{
63 cycle_t interval;
64 u64 tmp;
65
66 timekeeper.clock = clock;
67 clock->cycle_last = clock->read(clock);
68
69 /* Do the ns -> cycle conversion first, using original mult */
70 tmp = NTP_INTERVAL_LENGTH;
71 tmp <<= clock->shift;
0a544198
MS
72 tmp += clock->mult/2;
73 do_div(tmp, clock->mult);
155ec602
MS
74 if (tmp == 0)
75 tmp = 1;
76
77 interval = (cycle_t) tmp;
78 timekeeper.cycle_interval = interval;
79
80 /* Go back from cycles -> shifted ns */
81 timekeeper.xtime_interval = (u64) interval * clock->mult;
82 timekeeper.raw_interval =
0a544198 83 ((u64) interval * clock->mult) >> clock->shift;
155ec602
MS
84
85 timekeeper.xtime_nsec = 0;
23ce7211 86 timekeeper.shift = clock->shift;
155ec602
MS
87
88 timekeeper.ntp_error = 0;
23ce7211 89 timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
0a544198
MS
90
91 /*
92 * The timekeeper keeps its own mult values for the currently
93 * active clocksource. These value will be adjusted via NTP
94 * to counteract clock drifting.
95 */
96 timekeeper.mult = clock->mult;
155ec602 97}
8524070b 98
2ba2a305
MS
99/* Timekeeper helper functions. */
100static inline s64 timekeeping_get_ns(void)
101{
102 cycle_t cycle_now, cycle_delta;
103 struct clocksource *clock;
104
105 /* read clocksource: */
106 clock = timekeeper.clock;
107 cycle_now = clock->read(clock);
108
109 /* calculate the delta since the last update_wall_time: */
110 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
111
112 /* return delta convert to nanoseconds using ntp adjusted mult. */
113 return clocksource_cyc2ns(cycle_delta, timekeeper.mult,
114 timekeeper.shift);
115}
116
117static inline s64 timekeeping_get_ns_raw(void)
118{
119 cycle_t cycle_now, cycle_delta;
120 struct clocksource *clock;
121
122 /* read clocksource: */
123 clock = timekeeper.clock;
124 cycle_now = clock->read(clock);
125
126 /* calculate the delta since the last update_wall_time: */
127 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
128
129 /* return delta convert to nanoseconds using ntp adjusted mult. */
130 return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
131}
132
8524070b 133/*
134 * This read-write spinlock protects us from races in SMP while
dce48a84 135 * playing with xtime.
8524070b 136 */
ba2a631b 137__cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
8524070b 138
139
140/*
141 * The current time
142 * wall_to_monotonic is what we need to add to xtime (or xtime corrected
143 * for sub jiffie times) to get to monotonic time. Monotonic is pegged
144 * at zero at system boot time, so wall_to_monotonic will be negative,
145 * however, we will ALWAYS keep the tv_nsec part positive so we can use
146 * the usual normalization.
7c3f1a57
TJ
147 *
148 * wall_to_monotonic is moved after resume from suspend for the monotonic
149 * time not to jump. We need to add total_sleep_time to wall_to_monotonic
150 * to get the real boot based time offset.
151 *
152 * - wall_to_monotonic is no longer the boot time, getboottime must be
153 * used instead.
8524070b 154 */
155struct timespec xtime __attribute__ ((aligned (16)));
156struct timespec wall_to_monotonic __attribute__ ((aligned (16)));
7c3f1a57 157static unsigned long total_sleep_time; /* seconds */
8524070b 158
155ec602
MS
159/*
160 * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock.
161 */
162struct timespec raw_time;
163
1c5745aa
TG
164/* flag for if timekeeping is suspended */
165int __read_mostly timekeeping_suspended;
166
17c38b74 167static struct timespec xtime_cache __attribute__ ((aligned (16)));
1001d0a9 168void update_xtime_cache(u64 nsec)
17c38b74 169{
170 xtime_cache = xtime;
171 timespec_add_ns(&xtime_cache, nsec);
172}
17c38b74 173
31089c13
JS
174/* must hold xtime_lock */
175void timekeeping_leap_insert(int leapsecond)
176{
177 xtime.tv_sec += leapsecond;
178 wall_to_monotonic.tv_sec -= leapsecond;
155ec602 179 update_vsyscall(&xtime, timekeeper.clock);
31089c13 180}
8524070b 181
182#ifdef CONFIG_GENERIC_TIME
75c5158f 183
8524070b 184/**
155ec602 185 * timekeeping_forward_now - update clock to the current time
8524070b 186 *
9a055117
RZ
187 * Forward the current clock to update its state since the last call to
188 * update_wall_time(). This is useful before significant clock changes,
189 * as it avoids having to deal with this time offset explicitly.
8524070b 190 */
155ec602 191static void timekeeping_forward_now(void)
8524070b 192{
193 cycle_t cycle_now, cycle_delta;
155ec602 194 struct clocksource *clock;
9a055117 195 s64 nsec;
8524070b 196
155ec602 197 clock = timekeeper.clock;
a0f7d48b 198 cycle_now = clock->read(clock);
8524070b 199 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
9a055117 200 clock->cycle_last = cycle_now;
8524070b 201
0a544198
MS
202 nsec = clocksource_cyc2ns(cycle_delta, timekeeper.mult,
203 timekeeper.shift);
7d27558c 204
205 /* If arch requires, add in gettimeoffset() */
206 nsec += arch_gettimeoffset();
207
9a055117 208 timespec_add_ns(&xtime, nsec);
2d42244a 209
0a544198 210 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
155ec602 211 timespec_add_ns(&raw_time, nsec);
8524070b 212}
213
214/**
efd9ac86 215 * getnstimeofday - Returns the time of day in a timespec
8524070b 216 * @ts: pointer to the timespec to be set
217 *
efd9ac86 218 * Returns the time of day in a timespec.
8524070b 219 */
efd9ac86 220void getnstimeofday(struct timespec *ts)
8524070b 221{
222 unsigned long seq;
223 s64 nsecs;
224
1c5745aa
TG
225 WARN_ON(timekeeping_suspended);
226
8524070b 227 do {
228 seq = read_seqbegin(&xtime_lock);
229
230 *ts = xtime;
2ba2a305 231 nsecs = timekeeping_get_ns();
8524070b 232
7d27558c 233 /* If arch requires, add in gettimeoffset() */
234 nsecs += arch_gettimeoffset();
235
8524070b 236 } while (read_seqretry(&xtime_lock, seq));
237
238 timespec_add_ns(ts, nsecs);
239}
240
8524070b 241EXPORT_SYMBOL(getnstimeofday);
242
951ed4d3
MS
243ktime_t ktime_get(void)
244{
951ed4d3
MS
245 unsigned int seq;
246 s64 secs, nsecs;
247
248 WARN_ON(timekeeping_suspended);
249
250 do {
251 seq = read_seqbegin(&xtime_lock);
252 secs = xtime.tv_sec + wall_to_monotonic.tv_sec;
253 nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec;
2ba2a305 254 nsecs += timekeeping_get_ns();
951ed4d3
MS
255
256 } while (read_seqretry(&xtime_lock, seq));
257 /*
258 * Use ktime_set/ktime_add_ns to create a proper ktime on
259 * 32-bit architectures without CONFIG_KTIME_SCALAR.
260 */
261 return ktime_add_ns(ktime_set(secs, 0), nsecs);
262}
263EXPORT_SYMBOL_GPL(ktime_get);
264
265/**
266 * ktime_get_ts - get the monotonic clock in timespec format
267 * @ts: pointer to timespec variable
268 *
269 * The function calculates the monotonic clock from the realtime
270 * clock and the wall_to_monotonic offset and stores the result
271 * in normalized timespec format in the variable pointed to by @ts.
272 */
273void ktime_get_ts(struct timespec *ts)
274{
951ed4d3
MS
275 struct timespec tomono;
276 unsigned int seq;
277 s64 nsecs;
278
279 WARN_ON(timekeeping_suspended);
280
281 do {
282 seq = read_seqbegin(&xtime_lock);
283 *ts = xtime;
284 tomono = wall_to_monotonic;
2ba2a305 285 nsecs = timekeeping_get_ns();
951ed4d3
MS
286
287 } while (read_seqretry(&xtime_lock, seq));
288
289 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
290 ts->tv_nsec + tomono.tv_nsec + nsecs);
291}
292EXPORT_SYMBOL_GPL(ktime_get_ts);
293
8524070b 294/**
295 * do_gettimeofday - Returns the time of day in a timeval
296 * @tv: pointer to the timeval to be set
297 *
efd9ac86 298 * NOTE: Users should be converted to using getnstimeofday()
8524070b 299 */
300void do_gettimeofday(struct timeval *tv)
301{
302 struct timespec now;
303
efd9ac86 304 getnstimeofday(&now);
8524070b 305 tv->tv_sec = now.tv_sec;
306 tv->tv_usec = now.tv_nsec/1000;
307}
308
309EXPORT_SYMBOL(do_gettimeofday);
310/**
311 * do_settimeofday - Sets the time of day
312 * @tv: pointer to the timespec variable containing the new time
313 *
314 * Sets the time of day to the new time and update NTP and notify hrtimers
315 */
316int do_settimeofday(struct timespec *tv)
317{
9a055117 318 struct timespec ts_delta;
8524070b 319 unsigned long flags;
8524070b 320
321 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
322 return -EINVAL;
323
324 write_seqlock_irqsave(&xtime_lock, flags);
325
155ec602 326 timekeeping_forward_now();
9a055117
RZ
327
328 ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec;
329 ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec;
330 wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta);
8524070b 331
9a055117 332 xtime = *tv;
8524070b 333
1001d0a9 334 update_xtime_cache(0);
8524070b 335
155ec602 336 timekeeper.ntp_error = 0;
8524070b 337 ntp_clear();
338
155ec602 339 update_vsyscall(&xtime, timekeeper.clock);
8524070b 340
341 write_sequnlock_irqrestore(&xtime_lock, flags);
342
343 /* signal hrtimers about time change */
344 clock_was_set();
345
346 return 0;
347}
348
349EXPORT_SYMBOL(do_settimeofday);
350
351/**
352 * change_clocksource - Swaps clocksources if a new one is available
353 *
354 * Accumulates current time interval and initializes new clocksource
355 */
75c5158f 356static int change_clocksource(void *data)
8524070b 357{
4614e6ad 358 struct clocksource *new, *old;
8524070b 359
75c5158f 360 new = (struct clocksource *) data;
8524070b 361
155ec602 362 timekeeping_forward_now();
75c5158f
MS
363 if (!new->enable || new->enable(new) == 0) {
364 old = timekeeper.clock;
365 timekeeper_setup_internals(new);
366 if (old->disable)
367 old->disable(old);
368 }
369 return 0;
370}
8524070b 371
75c5158f
MS
372/**
373 * timekeeping_notify - Install a new clock source
374 * @clock: pointer to the clock source
375 *
376 * This function is called from clocksource.c after a new, better clock
377 * source has been registered. The caller holds the clocksource_mutex.
378 */
379void timekeeping_notify(struct clocksource *clock)
380{
381 if (timekeeper.clock == clock)
4614e6ad 382 return;
75c5158f 383 stop_machine(change_clocksource, clock, NULL);
8524070b 384 tick_clock_notify();
8524070b 385}
75c5158f 386
a40f262c 387#else /* GENERIC_TIME */
75c5158f 388
155ec602 389static inline void timekeeping_forward_now(void) { }
a40f262c
TG
390
391/**
392 * ktime_get - get the monotonic time in ktime_t format
393 *
394 * returns the time in ktime_t format
395 */
396ktime_t ktime_get(void)
397{
398 struct timespec now;
399
400 ktime_get_ts(&now);
401
402 return timespec_to_ktime(now);
403}
404EXPORT_SYMBOL_GPL(ktime_get);
405
406/**
407 * ktime_get_ts - get the monotonic clock in timespec format
408 * @ts: pointer to timespec variable
409 *
410 * The function calculates the monotonic clock from the realtime
411 * clock and the wall_to_monotonic offset and stores the result
412 * in normalized timespec format in the variable pointed to by @ts.
413 */
414void ktime_get_ts(struct timespec *ts)
415{
416 struct timespec tomono;
417 unsigned long seq;
418
419 do {
420 seq = read_seqbegin(&xtime_lock);
421 getnstimeofday(ts);
422 tomono = wall_to_monotonic;
423
424 } while (read_seqretry(&xtime_lock, seq));
425
426 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
427 ts->tv_nsec + tomono.tv_nsec);
428}
429EXPORT_SYMBOL_GPL(ktime_get_ts);
75c5158f 430
a40f262c
TG
431#endif /* !GENERIC_TIME */
432
433/**
434 * ktime_get_real - get the real (wall-) time in ktime_t format
435 *
436 * returns the time in ktime_t format
437 */
438ktime_t ktime_get_real(void)
439{
440 struct timespec now;
441
442 getnstimeofday(&now);
443
444 return timespec_to_ktime(now);
445}
446EXPORT_SYMBOL_GPL(ktime_get_real);
8524070b 447
2d42244a
JS
448/**
449 * getrawmonotonic - Returns the raw monotonic time in a timespec
450 * @ts: pointer to the timespec to be set
451 *
452 * Returns the raw monotonic time (completely un-modified by ntp)
453 */
454void getrawmonotonic(struct timespec *ts)
455{
456 unsigned long seq;
457 s64 nsecs;
2d42244a
JS
458
459 do {
460 seq = read_seqbegin(&xtime_lock);
2ba2a305 461 nsecs = timekeeping_get_ns_raw();
155ec602 462 *ts = raw_time;
2d42244a
JS
463
464 } while (read_seqretry(&xtime_lock, seq));
465
466 timespec_add_ns(ts, nsecs);
467}
468EXPORT_SYMBOL(getrawmonotonic);
469
470
8524070b 471/**
cf4fc6cb 472 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
8524070b 473 */
cf4fc6cb 474int timekeeping_valid_for_hres(void)
8524070b 475{
476 unsigned long seq;
477 int ret;
478
479 do {
480 seq = read_seqbegin(&xtime_lock);
481
155ec602 482 ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
8524070b 483
484 } while (read_seqretry(&xtime_lock, seq));
485
486 return ret;
487}
488
489/**
490 * read_persistent_clock - Return time in seconds from the persistent clock.
491 *
492 * Weak dummy function for arches that do not yet support it.
493 * Returns seconds from epoch using the battery backed persistent clock.
494 * Returns zero if unsupported.
495 *
496 * XXX - Do be sure to remove it once all arches implement it.
497 */
498unsigned long __attribute__((weak)) read_persistent_clock(void)
499{
500 return 0;
501}
502
503/*
504 * timekeeping_init - Initializes the clocksource and common timekeeping values
505 */
506void __init timekeeping_init(void)
507{
155ec602 508 struct clocksource *clock;
8524070b 509 unsigned long flags;
510 unsigned long sec = read_persistent_clock();
511
512 write_seqlock_irqsave(&xtime_lock, flags);
513
7dffa3c6 514 ntp_init();
8524070b 515
f1b82746 516 clock = clocksource_default_clock();
a0f7d48b
MS
517 if (clock->enable)
518 clock->enable(clock);
155ec602 519 timekeeper_setup_internals(clock);
8524070b 520
521 xtime.tv_sec = sec;
522 xtime.tv_nsec = 0;
155ec602
MS
523 raw_time.tv_sec = 0;
524 raw_time.tv_nsec = 0;
8524070b 525 set_normalized_timespec(&wall_to_monotonic,
526 -xtime.tv_sec, -xtime.tv_nsec);
1001d0a9 527 update_xtime_cache(0);
7c3f1a57 528 total_sleep_time = 0;
8524070b 529 write_sequnlock_irqrestore(&xtime_lock, flags);
530}
531
8524070b 532/* time in seconds when suspend began */
533static unsigned long timekeeping_suspend_time;
534
535/**
536 * timekeeping_resume - Resumes the generic timekeeping subsystem.
537 * @dev: unused
538 *
539 * This is for the generic clocksource timekeeping.
540 * xtime/wall_to_monotonic/jiffies/etc are
541 * still managed by arch specific suspend/resume code.
542 */
543static int timekeeping_resume(struct sys_device *dev)
544{
545 unsigned long flags;
546 unsigned long now = read_persistent_clock();
547
d10ff3fb
TG
548 clocksource_resume();
549
8524070b 550 write_seqlock_irqsave(&xtime_lock, flags);
551
552 if (now && (now > timekeeping_suspend_time)) {
553 unsigned long sleep_length = now - timekeeping_suspend_time;
554
555 xtime.tv_sec += sleep_length;
556 wall_to_monotonic.tv_sec -= sleep_length;
7c3f1a57 557 total_sleep_time += sleep_length;
8524070b 558 }
1001d0a9 559 update_xtime_cache(0);
8524070b 560 /* re-base the last cycle value */
155ec602
MS
561 timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
562 timekeeper.ntp_error = 0;
8524070b 563 timekeeping_suspended = 0;
564 write_sequnlock_irqrestore(&xtime_lock, flags);
565
566 touch_softlockup_watchdog();
567
568 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
569
570 /* Resume hrtimers */
571 hres_timers_resume();
572
573 return 0;
574}
575
576static int timekeeping_suspend(struct sys_device *dev, pm_message_t state)
577{
578 unsigned long flags;
579
3be90950
TG
580 timekeeping_suspend_time = read_persistent_clock();
581
8524070b 582 write_seqlock_irqsave(&xtime_lock, flags);
155ec602 583 timekeeping_forward_now();
8524070b 584 timekeeping_suspended = 1;
8524070b 585 write_sequnlock_irqrestore(&xtime_lock, flags);
586
587 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
588
589 return 0;
590}
591
592/* sysfs resume/suspend bits for timekeeping */
593static struct sysdev_class timekeeping_sysclass = {
af5ca3f4 594 .name = "timekeeping",
8524070b 595 .resume = timekeeping_resume,
596 .suspend = timekeeping_suspend,
8524070b 597};
598
599static struct sys_device device_timer = {
600 .id = 0,
601 .cls = &timekeeping_sysclass,
602};
603
604static int __init timekeeping_init_device(void)
605{
606 int error = sysdev_class_register(&timekeeping_sysclass);
607 if (!error)
608 error = sysdev_register(&device_timer);
609 return error;
610}
611
612device_initcall(timekeeping_init_device);
613
614/*
615 * If the error is already larger, we look ahead even further
616 * to compensate for late or lost adjustments.
617 */
155ec602 618static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
8524070b 619 s64 *offset)
620{
621 s64 tick_error, i;
622 u32 look_ahead, adj;
623 s32 error2, mult;
624
625 /*
626 * Use the current error value to determine how much to look ahead.
627 * The larger the error the slower we adjust for it to avoid problems
628 * with losing too many ticks, otherwise we would overadjust and
629 * produce an even larger error. The smaller the adjustment the
630 * faster we try to adjust for it, as lost ticks can do less harm
3eb05676 631 * here. This is tuned so that an error of about 1 msec is adjusted
8524070b 632 * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
633 */
155ec602 634 error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
8524070b 635 error2 = abs(error2);
636 for (look_ahead = 0; error2 > 0; look_ahead++)
637 error2 >>= 2;
638
639 /*
640 * Now calculate the error in (1 << look_ahead) ticks, but first
641 * remove the single look ahead already included in the error.
642 */
23ce7211 643 tick_error = tick_length >> (timekeeper.ntp_error_shift + 1);
155ec602 644 tick_error -= timekeeper.xtime_interval >> 1;
8524070b 645 error = ((error - tick_error) >> look_ahead) + tick_error;
646
647 /* Finally calculate the adjustment shift value. */
648 i = *interval;
649 mult = 1;
650 if (error < 0) {
651 error = -error;
652 *interval = -*interval;
653 *offset = -*offset;
654 mult = -1;
655 }
656 for (adj = 0; error > i; adj++)
657 error >>= 1;
658
659 *interval <<= adj;
660 *offset <<= adj;
661 return mult << adj;
662}
663
664/*
665 * Adjust the multiplier to reduce the error value,
666 * this is optimized for the most common adjustments of -1,0,1,
667 * for other values we can do a bit more work.
668 */
155ec602 669static void timekeeping_adjust(s64 offset)
8524070b 670{
155ec602 671 s64 error, interval = timekeeper.cycle_interval;
8524070b 672 int adj;
673
23ce7211 674 error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1);
8524070b 675 if (error > interval) {
676 error >>= 2;
677 if (likely(error <= interval))
678 adj = 1;
679 else
155ec602 680 adj = timekeeping_bigadjust(error, &interval, &offset);
8524070b 681 } else if (error < -interval) {
682 error >>= 2;
683 if (likely(error >= -interval)) {
684 adj = -1;
685 interval = -interval;
686 offset = -offset;
687 } else
155ec602 688 adj = timekeeping_bigadjust(error, &interval, &offset);
8524070b 689 } else
690 return;
691
0a544198 692 timekeeper.mult += adj;
155ec602
MS
693 timekeeper.xtime_interval += interval;
694 timekeeper.xtime_nsec -= offset;
695 timekeeper.ntp_error -= (interval - offset) <<
23ce7211 696 timekeeper.ntp_error_shift;
8524070b 697}
698
699/**
700 * update_wall_time - Uses the current clocksource to increment the wall time
701 *
702 * Called from the timer interrupt, must hold a write on xtime_lock.
703 */
704void update_wall_time(void)
705{
155ec602 706 struct clocksource *clock;
8524070b 707 cycle_t offset;
23ce7211 708 u64 nsecs;
8524070b 709
710 /* Make sure we're fully resumed: */
711 if (unlikely(timekeeping_suspended))
712 return;
713
155ec602 714 clock = timekeeper.clock;
8524070b 715#ifdef CONFIG_GENERIC_TIME
a0f7d48b 716 offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
8524070b 717#else
155ec602 718 offset = timekeeper.cycle_interval;
8524070b 719#endif
23ce7211 720 timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift;
8524070b 721
722 /* normally this loop will run just once, however in the
723 * case of lost or late ticks, it will accumulate correctly.
724 */
155ec602 725 while (offset >= timekeeper.cycle_interval) {
23ce7211 726 u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift;
155ec602 727
8524070b 728 /* accumulate one interval */
155ec602
MS
729 offset -= timekeeper.cycle_interval;
730 clock->cycle_last += timekeeper.cycle_interval;
8524070b 731
155ec602
MS
732 timekeeper.xtime_nsec += timekeeper.xtime_interval;
733 if (timekeeper.xtime_nsec >= nsecps) {
734 timekeeper.xtime_nsec -= nsecps;
8524070b 735 xtime.tv_sec++;
736 second_overflow();
737 }
738
155ec602
MS
739 raw_time.tv_nsec += timekeeper.raw_interval;
740 if (raw_time.tv_nsec >= NSEC_PER_SEC) {
741 raw_time.tv_nsec -= NSEC_PER_SEC;
742 raw_time.tv_sec++;
2d42244a
JS
743 }
744
8524070b 745 /* accumulate error between NTP and clock interval */
155ec602
MS
746 timekeeper.ntp_error += tick_length;
747 timekeeper.ntp_error -= timekeeper.xtime_interval <<
23ce7211 748 timekeeper.ntp_error_shift;
8524070b 749 }
750
751 /* correct the clock when NTP error is too big */
155ec602 752 timekeeping_adjust(offset);
8524070b 753
6c9bacb4 754 /*
755 * Since in the loop above, we accumulate any amount of time
756 * in xtime_nsec over a second into xtime.tv_sec, its possible for
757 * xtime_nsec to be fairly small after the loop. Further, if we're
155ec602 758 * slightly speeding the clocksource up in timekeeping_adjust(),
6c9bacb4 759 * its possible the required corrective factor to xtime_nsec could
760 * cause it to underflow.
761 *
762 * Now, we cannot simply roll the accumulated second back, since
763 * the NTP subsystem has been notified via second_overflow. So
764 * instead we push xtime_nsec forward by the amount we underflowed,
765 * and add that amount into the error.
766 *
767 * We'll correct this error next time through this function, when
768 * xtime_nsec is not as small.
769 */
155ec602
MS
770 if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
771 s64 neg = -(s64)timekeeper.xtime_nsec;
772 timekeeper.xtime_nsec = 0;
23ce7211 773 timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
6c9bacb4 774 }
775
5cd1c9c5
RZ
776 /* store full nanoseconds into xtime after rounding it up and
777 * add the remainder to the error difference.
778 */
23ce7211
MS
779 xtime.tv_nsec = ((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1;
780 timekeeper.xtime_nsec -= (s64) xtime.tv_nsec << timekeeper.shift;
781 timekeeper.ntp_error += timekeeper.xtime_nsec <<
782 timekeeper.ntp_error_shift;
8524070b 783
0a544198 784 nsecs = clocksource_cyc2ns(offset, timekeeper.mult, timekeeper.shift);
155ec602 785 update_xtime_cache(nsecs);
17c38b74 786
8524070b 787 /* check to see if there is a new clocksource to use */
155ec602 788 update_vsyscall(&xtime, timekeeper.clock);
8524070b 789}
7c3f1a57
TJ
790
791/**
792 * getboottime - Return the real time of system boot.
793 * @ts: pointer to the timespec to be set
794 *
795 * Returns the time of day in a timespec.
796 *
797 * This is based on the wall_to_monotonic offset and the total suspend
798 * time. Calls to settimeofday will affect the value returned (which
799 * basically means that however wrong your real time clock is at boot time,
800 * you get the right time here).
801 */
802void getboottime(struct timespec *ts)
803{
804 set_normalized_timespec(ts,
805 - (wall_to_monotonic.tv_sec + total_sleep_time),
806 - wall_to_monotonic.tv_nsec);
807}
808
809/**
810 * monotonic_to_bootbased - Convert the monotonic time to boot based.
811 * @ts: pointer to the timespec to be converted
812 */
813void monotonic_to_bootbased(struct timespec *ts)
814{
815 ts->tv_sec += total_sleep_time;
816}
2c6b47de 817
17c38b74 818unsigned long get_seconds(void)
819{
820 return xtime_cache.tv_sec;
821}
822EXPORT_SYMBOL(get_seconds);
823
824
2c6b47de 825struct timespec current_kernel_time(void)
826{
827 struct timespec now;
828 unsigned long seq;
829
830 do {
831 seq = read_seqbegin(&xtime_lock);
832
17c38b74 833 now = xtime_cache;
2c6b47de 834 } while (read_seqretry(&xtime_lock, seq));
835
836 return now;
837}
2c6b47de 838EXPORT_SYMBOL(current_kernel_time);
This page took 0.287005 seconds and 5 git commands to generate.