Commit | Line | Data |
---|---|---|
112f38a4 RK |
1 | /* |
2 | * sched_clock.c: support for extending counters to full 64-bit ns counter | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | #include <linux/clocksource.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/jiffies.h> | |
a08ca5d1 | 11 | #include <linux/ktime.h> |
112f38a4 | 12 | #include <linux/kernel.h> |
a42c3629 | 13 | #include <linux/moduleparam.h> |
112f38a4 | 14 | #include <linux/sched.h> |
f153d017 | 15 | #include <linux/syscore_ops.h> |
a08ca5d1 | 16 | #include <linux/hrtimer.h> |
38ff87f7 | 17 | #include <linux/sched_clock.h> |
85c3d2dd | 18 | #include <linux/seqlock.h> |
e7e3ff1b | 19 | #include <linux/bitops.h> |
112f38a4 | 20 | |
2f0778af | 21 | struct clock_data { |
a08ca5d1 | 22 | ktime_t wrap_kt; |
2f0778af | 23 | u64 epoch_ns; |
e7e3ff1b | 24 | u64 epoch_cyc; |
85c3d2dd | 25 | seqcount_t seq; |
c115739d | 26 | unsigned long rate; |
2f0778af MZ |
27 | u32 mult; |
28 | u32 shift; | |
237ec6f2 | 29 | bool suspended; |
2f0778af MZ |
30 | }; |
31 | ||
a08ca5d1 | 32 | static struct hrtimer sched_clock_timer; |
a42c3629 RK |
33 | static int irqtime = -1; |
34 | ||
35 | core_param(irqtime, irqtime, int, 0400); | |
2f0778af MZ |
36 | |
37 | static struct clock_data cd = { | |
38 | .mult = NSEC_PER_SEC / HZ, | |
39 | }; | |
40 | ||
e7e3ff1b | 41 | static u64 __read_mostly sched_clock_mask; |
2f0778af | 42 | |
e7e3ff1b | 43 | static u64 notrace jiffy_sched_clock_read(void) |
2f0778af | 44 | { |
e7e3ff1b SB |
45 | /* |
46 | * We don't need to use get_jiffies_64 on 32-bit arches here | |
47 | * because we register with BITS_PER_LONG | |
48 | */ | |
49 | return (u64)(jiffies - INITIAL_JIFFIES); | |
50 | } | |
51 | ||
52 | static u32 __read_mostly (*read_sched_clock_32)(void); | |
53 | ||
54 | static u64 notrace read_sched_clock_32_wrapper(void) | |
55 | { | |
56 | return read_sched_clock_32(); | |
2f0778af MZ |
57 | } |
58 | ||
e7e3ff1b | 59 | static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; |
2f0778af | 60 | |
cea15092 | 61 | static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) |
2f0778af MZ |
62 | { |
63 | return (cyc * mult) >> shift; | |
64 | } | |
65 | ||
b4042cea | 66 | unsigned long long notrace sched_clock(void) |
2f0778af MZ |
67 | { |
68 | u64 epoch_ns; | |
e7e3ff1b SB |
69 | u64 epoch_cyc; |
70 | u64 cyc; | |
85c3d2dd | 71 | unsigned long seq; |
336ae118 SB |
72 | |
73 | if (cd.suspended) | |
74 | return cd.epoch_ns; | |
2f0778af | 75 | |
2f0778af | 76 | do { |
7a06c41c | 77 | seq = raw_read_seqcount_begin(&cd.seq); |
2f0778af | 78 | epoch_cyc = cd.epoch_cyc; |
2f0778af | 79 | epoch_ns = cd.epoch_ns; |
85c3d2dd | 80 | } while (read_seqcount_retry(&cd.seq, seq)); |
2f0778af | 81 | |
336ae118 SB |
82 | cyc = read_sched_clock(); |
83 | cyc = (cyc - epoch_cyc) & sched_clock_mask; | |
84 | return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift); | |
2f0778af MZ |
85 | } |
86 | ||
87 | /* | |
88 | * Atomically update the sched_clock epoch. | |
89 | */ | |
90 | static void notrace update_sched_clock(void) | |
91 | { | |
92 | unsigned long flags; | |
e7e3ff1b | 93 | u64 cyc; |
2f0778af MZ |
94 | u64 ns; |
95 | ||
96 | cyc = read_sched_clock(); | |
97 | ns = cd.epoch_ns + | |
98 | cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, | |
99 | cd.mult, cd.shift); | |
85c3d2dd | 100 | |
2f0778af | 101 | raw_local_irq_save(flags); |
7a06c41c | 102 | raw_write_seqcount_begin(&cd.seq); |
2f0778af | 103 | cd.epoch_ns = ns; |
7c4e9ced | 104 | cd.epoch_cyc = cyc; |
7a06c41c | 105 | raw_write_seqcount_end(&cd.seq); |
2f0778af MZ |
106 | raw_local_irq_restore(flags); |
107 | } | |
112f38a4 | 108 | |
a08ca5d1 | 109 | static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) |
112f38a4 | 110 | { |
2f0778af | 111 | update_sched_clock(); |
a08ca5d1 SB |
112 | hrtimer_forward_now(hrt, cd.wrap_kt); |
113 | return HRTIMER_RESTART; | |
112f38a4 RK |
114 | } |
115 | ||
e7e3ff1b SB |
116 | void __init sched_clock_register(u64 (*read)(void), int bits, |
117 | unsigned long rate) | |
112f38a4 | 118 | { |
5ae8aabe SB |
119 | u64 res, wrap, new_mask, new_epoch, cyc, ns; |
120 | u32 new_mult, new_shift; | |
121 | ktime_t new_wrap_kt; | |
a08ca5d1 | 122 | unsigned long r; |
112f38a4 RK |
123 | char r_unit; |
124 | ||
c115739d RH |
125 | if (cd.rate > rate) |
126 | return; | |
127 | ||
2f0778af | 128 | WARN_ON(!irqs_disabled()); |
112f38a4 RK |
129 | |
130 | /* calculate the mult/shift to convert counter ticks to ns. */ | |
5ae8aabe SB |
131 | clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); |
132 | ||
133 | new_mask = CLOCKSOURCE_MASK(bits); | |
134 | ||
135 | /* calculate how many ns until we wrap */ | |
136 | wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask); | |
137 | new_wrap_kt = ns_to_ktime(wrap - (wrap >> 3)); | |
138 | ||
139 | /* update epoch for new counter and update epoch_ns from old counter*/ | |
140 | new_epoch = read(); | |
141 | cyc = read_sched_clock(); | |
142 | ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, | |
143 | cd.mult, cd.shift); | |
144 | ||
145 | raw_write_seqcount_begin(&cd.seq); | |
146 | read_sched_clock = read; | |
147 | sched_clock_mask = new_mask; | |
148 | cd.rate = rate; | |
149 | cd.wrap_kt = new_wrap_kt; | |
150 | cd.mult = new_mult; | |
151 | cd.shift = new_shift; | |
152 | cd.epoch_cyc = new_epoch; | |
153 | cd.epoch_ns = ns; | |
154 | raw_write_seqcount_end(&cd.seq); | |
112f38a4 RK |
155 | |
156 | r = rate; | |
157 | if (r >= 4000000) { | |
158 | r /= 1000000; | |
159 | r_unit = 'M'; | |
2f0778af | 160 | } else if (r >= 1000) { |
112f38a4 RK |
161 | r /= 1000; |
162 | r_unit = 'k'; | |
2f0778af MZ |
163 | } else |
164 | r_unit = ' '; | |
112f38a4 | 165 | |
112f38a4 | 166 | /* calculate the ns resolution of this counter */ |
5ae8aabe SB |
167 | res = cyc_to_ns(1ULL, new_mult, new_shift); |
168 | ||
a08ca5d1 SB |
169 | pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", |
170 | bits, r, r_unit, res, wrap); | |
112f38a4 | 171 | |
a42c3629 RK |
172 | /* Enable IRQ time accounting if we have a fast enough sched_clock */ |
173 | if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) | |
174 | enable_sched_clock_irqtime(); | |
175 | ||
2f0778af MZ |
176 | pr_debug("Registered %pF as sched_clock source\n", read); |
177 | } | |
178 | ||
e7e3ff1b SB |
179 | void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) |
180 | { | |
181 | read_sched_clock_32 = read; | |
182 | sched_clock_register(read_sched_clock_32_wrapper, bits, rate); | |
183 | } | |
184 | ||
211baa70 RK |
185 | void __init sched_clock_postinit(void) |
186 | { | |
2f0778af MZ |
187 | /* |
188 | * If no sched_clock function has been provided at that point, | |
189 | * make it the final one one. | |
190 | */ | |
191 | if (read_sched_clock == jiffy_sched_clock_read) | |
e7e3ff1b | 192 | sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); |
2f0778af | 193 | |
a08ca5d1 SB |
194 | update_sched_clock(); |
195 | ||
196 | /* | |
197 | * Start the timer to keep sched_clock() properly updated and | |
198 | * sets the initial epoch. | |
199 | */ | |
200 | hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | |
201 | sched_clock_timer.function = sched_clock_poll; | |
202 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); | |
211baa70 | 203 | } |
f153d017 RK |
204 | |
205 | static int sched_clock_suspend(void) | |
206 | { | |
a08ca5d1 | 207 | sched_clock_poll(&sched_clock_timer); |
6a4dae5e | 208 | cd.suspended = true; |
f153d017 RK |
209 | return 0; |
210 | } | |
211 | ||
237ec6f2 CC |
212 | static void sched_clock_resume(void) |
213 | { | |
6a4dae5e | 214 | cd.epoch_cyc = read_sched_clock(); |
6a4dae5e | 215 | cd.suspended = false; |
237ec6f2 CC |
216 | } |
217 | ||
f153d017 RK |
218 | static struct syscore_ops sched_clock_ops = { |
219 | .suspend = sched_clock_suspend, | |
237ec6f2 | 220 | .resume = sched_clock_resume, |
f153d017 RK |
221 | }; |
222 | ||
223 | static int __init sched_clock_syscore_init(void) | |
224 | { | |
225 | register_syscore_ops(&sched_clock_ops); | |
226 | return 0; | |
227 | } | |
228 | device_initcall(sched_clock_syscore_init); |