Commit | Line | Data |
---|---|---|
112f38a4 RK |
1 | /* |
2 | * sched_clock.c: support for extending counters to full 64-bit ns counter | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | #include <linux/clocksource.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/jiffies.h> | |
a08ca5d1 | 11 | #include <linux/ktime.h> |
112f38a4 | 12 | #include <linux/kernel.h> |
a42c3629 | 13 | #include <linux/moduleparam.h> |
112f38a4 | 14 | #include <linux/sched.h> |
f153d017 | 15 | #include <linux/syscore_ops.h> |
a08ca5d1 | 16 | #include <linux/hrtimer.h> |
38ff87f7 | 17 | #include <linux/sched_clock.h> |
85c3d2dd | 18 | #include <linux/seqlock.h> |
e7e3ff1b | 19 | #include <linux/bitops.h> |
112f38a4 | 20 | |
2f0778af | 21 | struct clock_data { |
a08ca5d1 | 22 | ktime_t wrap_kt; |
2f0778af | 23 | u64 epoch_ns; |
e7e3ff1b | 24 | u64 epoch_cyc; |
85c3d2dd | 25 | seqcount_t seq; |
c115739d | 26 | unsigned long rate; |
2f0778af MZ |
27 | u32 mult; |
28 | u32 shift; | |
237ec6f2 | 29 | bool suspended; |
2f0778af MZ |
30 | }; |
31 | ||
a08ca5d1 | 32 | static struct hrtimer sched_clock_timer; |
a42c3629 RK |
33 | static int irqtime = -1; |
34 | ||
35 | core_param(irqtime, irqtime, int, 0400); | |
2f0778af MZ |
36 | |
37 | static struct clock_data cd = { | |
38 | .mult = NSEC_PER_SEC / HZ, | |
39 | }; | |
40 | ||
e7e3ff1b | 41 | static u64 __read_mostly sched_clock_mask; |
2f0778af | 42 | |
e7e3ff1b | 43 | static u64 notrace jiffy_sched_clock_read(void) |
2f0778af | 44 | { |
e7e3ff1b SB |
45 | /* |
46 | * We don't need to use get_jiffies_64 on 32-bit arches here | |
47 | * because we register with BITS_PER_LONG | |
48 | */ | |
49 | return (u64)(jiffies - INITIAL_JIFFIES); | |
50 | } | |
51 | ||
e7e3ff1b | 52 | static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; |
2f0778af | 53 | |
cea15092 | 54 | static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) |
2f0778af MZ |
55 | { |
56 | return (cyc * mult) >> shift; | |
57 | } | |
58 | ||
b4042cea | 59 | unsigned long long notrace sched_clock(void) |
2f0778af MZ |
60 | { |
61 | u64 epoch_ns; | |
e7e3ff1b SB |
62 | u64 epoch_cyc; |
63 | u64 cyc; | |
85c3d2dd | 64 | unsigned long seq; |
336ae118 SB |
65 | |
66 | if (cd.suspended) | |
67 | return cd.epoch_ns; | |
2f0778af | 68 | |
2f0778af | 69 | do { |
7a06c41c | 70 | seq = raw_read_seqcount_begin(&cd.seq); |
2f0778af | 71 | epoch_cyc = cd.epoch_cyc; |
2f0778af | 72 | epoch_ns = cd.epoch_ns; |
85c3d2dd | 73 | } while (read_seqcount_retry(&cd.seq, seq)); |
2f0778af | 74 | |
336ae118 SB |
75 | cyc = read_sched_clock(); |
76 | cyc = (cyc - epoch_cyc) & sched_clock_mask; | |
77 | return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift); | |
2f0778af MZ |
78 | } |
79 | ||
80 | /* | |
81 | * Atomically update the sched_clock epoch. | |
82 | */ | |
83 | static void notrace update_sched_clock(void) | |
84 | { | |
85 | unsigned long flags; | |
e7e3ff1b | 86 | u64 cyc; |
2f0778af MZ |
87 | u64 ns; |
88 | ||
89 | cyc = read_sched_clock(); | |
90 | ns = cd.epoch_ns + | |
91 | cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, | |
92 | cd.mult, cd.shift); | |
85c3d2dd | 93 | |
2f0778af | 94 | raw_local_irq_save(flags); |
7a06c41c | 95 | raw_write_seqcount_begin(&cd.seq); |
2f0778af | 96 | cd.epoch_ns = ns; |
7c4e9ced | 97 | cd.epoch_cyc = cyc; |
7a06c41c | 98 | raw_write_seqcount_end(&cd.seq); |
2f0778af MZ |
99 | raw_local_irq_restore(flags); |
100 | } | |
112f38a4 | 101 | |
a08ca5d1 | 102 | static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) |
112f38a4 | 103 | { |
2f0778af | 104 | update_sched_clock(); |
a08ca5d1 SB |
105 | hrtimer_forward_now(hrt, cd.wrap_kt); |
106 | return HRTIMER_RESTART; | |
112f38a4 RK |
107 | } |
108 | ||
e7e3ff1b SB |
109 | void __init sched_clock_register(u64 (*read)(void), int bits, |
110 | unsigned long rate) | |
112f38a4 | 111 | { |
5ae8aabe SB |
112 | u64 res, wrap, new_mask, new_epoch, cyc, ns; |
113 | u32 new_mult, new_shift; | |
114 | ktime_t new_wrap_kt; | |
a08ca5d1 | 115 | unsigned long r; |
112f38a4 RK |
116 | char r_unit; |
117 | ||
c115739d RH |
118 | if (cd.rate > rate) |
119 | return; | |
120 | ||
2f0778af | 121 | WARN_ON(!irqs_disabled()); |
112f38a4 RK |
122 | |
123 | /* calculate the mult/shift to convert counter ticks to ns. */ | |
5ae8aabe SB |
124 | clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); |
125 | ||
126 | new_mask = CLOCKSOURCE_MASK(bits); | |
127 | ||
128 | /* calculate how many ns until we wrap */ | |
129 | wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask); | |
130 | new_wrap_kt = ns_to_ktime(wrap - (wrap >> 3)); | |
131 | ||
132 | /* update epoch for new counter and update epoch_ns from old counter*/ | |
133 | new_epoch = read(); | |
134 | cyc = read_sched_clock(); | |
135 | ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, | |
136 | cd.mult, cd.shift); | |
137 | ||
138 | raw_write_seqcount_begin(&cd.seq); | |
139 | read_sched_clock = read; | |
140 | sched_clock_mask = new_mask; | |
141 | cd.rate = rate; | |
142 | cd.wrap_kt = new_wrap_kt; | |
143 | cd.mult = new_mult; | |
144 | cd.shift = new_shift; | |
145 | cd.epoch_cyc = new_epoch; | |
146 | cd.epoch_ns = ns; | |
147 | raw_write_seqcount_end(&cd.seq); | |
112f38a4 RK |
148 | |
149 | r = rate; | |
150 | if (r >= 4000000) { | |
151 | r /= 1000000; | |
152 | r_unit = 'M'; | |
2f0778af | 153 | } else if (r >= 1000) { |
112f38a4 RK |
154 | r /= 1000; |
155 | r_unit = 'k'; | |
2f0778af MZ |
156 | } else |
157 | r_unit = ' '; | |
112f38a4 | 158 | |
112f38a4 | 159 | /* calculate the ns resolution of this counter */ |
5ae8aabe SB |
160 | res = cyc_to_ns(1ULL, new_mult, new_shift); |
161 | ||
a08ca5d1 SB |
162 | pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", |
163 | bits, r, r_unit, res, wrap); | |
112f38a4 | 164 | |
a42c3629 RK |
165 | /* Enable IRQ time accounting if we have a fast enough sched_clock */ |
166 | if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) | |
167 | enable_sched_clock_irqtime(); | |
168 | ||
2f0778af MZ |
169 | pr_debug("Registered %pF as sched_clock source\n", read); |
170 | } | |
171 | ||
211baa70 RK |
172 | void __init sched_clock_postinit(void) |
173 | { | |
2f0778af MZ |
174 | /* |
175 | * If no sched_clock function has been provided at that point, | |
176 | * make it the final one one. | |
177 | */ | |
178 | if (read_sched_clock == jiffy_sched_clock_read) | |
e7e3ff1b | 179 | sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); |
2f0778af | 180 | |
a08ca5d1 SB |
181 | update_sched_clock(); |
182 | ||
183 | /* | |
184 | * Start the timer to keep sched_clock() properly updated and | |
185 | * sets the initial epoch. | |
186 | */ | |
187 | hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | |
188 | sched_clock_timer.function = sched_clock_poll; | |
189 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); | |
211baa70 | 190 | } |
f153d017 RK |
191 | |
192 | static int sched_clock_suspend(void) | |
193 | { | |
f723aa18 SB |
194 | update_sched_clock(); |
195 | hrtimer_cancel(&sched_clock_timer); | |
6a4dae5e | 196 | cd.suspended = true; |
f153d017 RK |
197 | return 0; |
198 | } | |
199 | ||
237ec6f2 CC |
200 | static void sched_clock_resume(void) |
201 | { | |
6a4dae5e | 202 | cd.epoch_cyc = read_sched_clock(); |
f723aa18 | 203 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); |
6a4dae5e | 204 | cd.suspended = false; |
237ec6f2 CC |
205 | } |
206 | ||
f153d017 RK |
207 | static struct syscore_ops sched_clock_ops = { |
208 | .suspend = sched_clock_suspend, | |
237ec6f2 | 209 | .resume = sched_clock_resume, |
f153d017 RK |
210 | }; |
211 | ||
212 | static int __init sched_clock_syscore_init(void) | |
213 | { | |
214 | register_syscore_ops(&sched_clock_ops); | |
215 | return 0; | |
216 | } | |
217 | device_initcall(sched_clock_syscore_init); |