Commit | Line | Data |
---|---|---|
250c2277 | 1 | /* |
835c34a1 | 2 | * check TSC synchronization. |
250c2277 TG |
3 | * |
4 | * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar | |
5 | * | |
6 | * We check whether all boot CPUs have their TSC's synchronized, | |
7 | * print a warning if not and turn off the TSC clock-source. | |
8 | * | |
9 | * The warp-check is point-to-point between two CPUs, the CPU | |
10 | * initiating the bootup is the 'source CPU', the freshly booting | |
11 | * CPU is the 'target CPU'. | |
12 | * | |
13 | * Only two CPUs may participate - they can enter in any order. | |
14 | * ( The serial nature of the boot logic and the CPU hotplug lock | |
15 | * protects against more than 2 CPUs entering this code. ) | |
16 | */ | |
17 | #include <linux/spinlock.h> | |
18 | #include <linux/kernel.h> | |
250c2277 TG |
19 | #include <linux/smp.h> |
20 | #include <linux/nmi.h> | |
21 | #include <asm/tsc.h> | |
22 | ||
23 | /* | |
24 | * Entry/exit counters that make sure that both CPUs | |
25 | * run the measurement code at once: | |
26 | */ | |
148f9bb8 PG |
27 | static atomic_t start_count; |
28 | static atomic_t stop_count; | |
250c2277 TG |
29 | |
30 | /* | |
31 | * We use a raw spinlock in this exceptional case, because | |
32 | * we want to have the fastest, inlined, non-debug version | |
33 | * of a critical section, to be able to prove TSC time-warps: | |
34 | */ | |
148f9bb8 | 35 | static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; |
643bec95 | 36 | |
148f9bb8 PG |
37 | static cycles_t last_tsc; |
38 | static cycles_t max_warp; | |
39 | static int nr_warps; | |
250c2277 TG |
40 | |
41 | /* | |
eee6946e AL |
42 | * TSC-warp measurement loop running on both CPUs. This is not called |
43 | * if there is no TSC. | |
250c2277 | 44 | */ |
148f9bb8 | 45 | static void check_tsc_warp(unsigned int timeout) |
250c2277 TG |
46 | { |
47 | cycles_t start, now, prev, end; | |
48 | int i; | |
49 | ||
eee6946e | 50 | start = rdtsc_ordered(); |
250c2277 | 51 | /* |
b0e5c779 | 52 | * The measurement runs for 'timeout' msecs: |
250c2277 | 53 | */ |
b0e5c779 | 54 | end = start + (cycles_t) tsc_khz * timeout; |
250c2277 TG |
55 | now = start; |
56 | ||
57 | for (i = 0; ; i++) { | |
58 | /* | |
59 | * We take the global lock, measure TSC, save the | |
60 | * previous TSC that was measured (possibly on | |
61 | * another CPU) and update the previous TSC timestamp. | |
62 | */ | |
0199c4e6 | 63 | arch_spin_lock(&sync_lock); |
250c2277 | 64 | prev = last_tsc; |
eee6946e | 65 | now = rdtsc_ordered(); |
250c2277 | 66 | last_tsc = now; |
0199c4e6 | 67 | arch_spin_unlock(&sync_lock); |
250c2277 TG |
68 | |
69 | /* | |
70 | * Be nice every now and then (and also check whether | |
df43510b | 71 | * measurement is done [we also insert a 10 million |
250c2277 TG |
72 | * loops safety exit, so we dont lock up in case the |
73 | * TSC readout is totally broken]): | |
74 | */ | |
75 | if (unlikely(!(i & 7))) { | |
df43510b | 76 | if (now > end || i > 10000000) |
250c2277 TG |
77 | break; |
78 | cpu_relax(); | |
79 | touch_nmi_watchdog(); | |
80 | } | |
81 | /* | |
82 | * Outside the critical section we can now see whether | |
83 | * we saw a time-warp of the TSC going backwards: | |
84 | */ | |
85 | if (unlikely(prev > now)) { | |
0199c4e6 | 86 | arch_spin_lock(&sync_lock); |
250c2277 TG |
87 | max_warp = max(max_warp, prev - now); |
88 | nr_warps++; | |
0199c4e6 | 89 | arch_spin_unlock(&sync_lock); |
250c2277 | 90 | } |
ad8ca495 | 91 | } |
bde78a79 AV |
92 | WARN(!(now-start), |
93 | "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", | |
ad8ca495 | 94 | now-start, end-start); |
250c2277 TG |
95 | } |
96 | ||
b0e5c779 SS |
97 | /* |
98 | * If the target CPU coming online doesn't have any of its core-siblings | |
99 | * online, a timeout of 20msec will be used for the TSC-warp measurement | |
100 | * loop. Otherwise a smaller timeout of 2msec will be used, as we have some | |
101 | * information about this socket already (and this information grows as we | |
102 | * have more and more logical-siblings in that socket). | |
103 | * | |
104 | * Ideally we should be able to skip the TSC sync check on the other | |
105 | * core-siblings, if the first logical CPU in a socket passed the sync test. | |
106 | * But as the TSC is per-logical CPU and can potentially be modified wrongly | |
107 | * by the bios, TSC sync test for smaller duration should be able | |
108 | * to catch such errors. Also this will catch the condition where all the | |
109 | * cores in the socket doesn't get reset at the same time. | |
110 | */ | |
111 | static inline unsigned int loop_timeout(int cpu) | |
112 | { | |
7d79a7bd | 113 | return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20; |
b0e5c779 SS |
114 | } |
115 | ||
250c2277 TG |
116 | /* |
117 | * Source CPU calls into this - it waits for the freshly booted | |
118 | * target CPU to arrive and then starts the measurement: | |
119 | */ | |
148f9bb8 | 120 | void check_tsc_sync_source(int cpu) |
250c2277 TG |
121 | { |
122 | int cpus = 2; | |
123 | ||
124 | /* | |
125 | * No need to check if we already know that the TSC is not | |
eee6946e | 126 | * synchronized or if we have no TSC. |
250c2277 TG |
127 | */ |
128 | if (unsynchronized_tsc()) | |
129 | return; | |
130 | ||
28a00184 | 131 | if (tsc_clocksource_reliable) { |
9b3660a5 MT |
132 | if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING) |
133 | pr_info( | |
134 | "Skipped synchronization checks as TSC is reliable.\n"); | |
eca0cd02 AK |
135 | return; |
136 | } | |
137 | ||
250c2277 TG |
138 | /* |
139 | * Reset it - in case this is a second bootup: | |
140 | */ | |
141 | atomic_set(&stop_count, 0); | |
142 | ||
143 | /* | |
144 | * Wait for the target to arrive: | |
145 | */ | |
146 | while (atomic_read(&start_count) != cpus-1) | |
147 | cpu_relax(); | |
148 | /* | |
149 | * Trigger the target to continue into the measurement too: | |
150 | */ | |
151 | atomic_inc(&start_count); | |
152 | ||
b0e5c779 | 153 | check_tsc_warp(loop_timeout(cpu)); |
250c2277 TG |
154 | |
155 | while (atomic_read(&stop_count) != cpus-1) | |
156 | cpu_relax(); | |
157 | ||
250c2277 | 158 | if (nr_warps) { |
9b3660a5 MT |
159 | pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n", |
160 | smp_processor_id(), cpu); | |
643bec95 IM |
161 | pr_warning("Measured %Ld cycles TSC warp between CPUs, " |
162 | "turning off TSC clock.\n", max_warp); | |
250c2277 | 163 | mark_tsc_unstable("check_tsc_sync_source failed"); |
250c2277 | 164 | } else { |
9b3660a5 MT |
165 | pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n", |
166 | smp_processor_id(), cpu); | |
250c2277 TG |
167 | } |
168 | ||
4c6b8b4d MG |
169 | /* |
170 | * Reset it - just in case we boot another CPU later: | |
171 | */ | |
172 | atomic_set(&start_count, 0); | |
173 | nr_warps = 0; | |
174 | max_warp = 0; | |
175 | last_tsc = 0; | |
176 | ||
250c2277 TG |
177 | /* |
178 | * Let the target continue with the bootup: | |
179 | */ | |
180 | atomic_inc(&stop_count); | |
181 | } | |
182 | ||
183 | /* | |
184 | * Freshly booted CPUs call into this: | |
185 | */ | |
148f9bb8 | 186 | void check_tsc_sync_target(void) |
250c2277 TG |
187 | { |
188 | int cpus = 2; | |
189 | ||
eee6946e | 190 | /* Also aborts if there is no TSC. */ |
28a00184 | 191 | if (unsynchronized_tsc() || tsc_clocksource_reliable) |
250c2277 TG |
192 | return; |
193 | ||
194 | /* | |
195 | * Register this CPU's participation and wait for the | |
196 | * source CPU to start the measurement: | |
197 | */ | |
198 | atomic_inc(&start_count); | |
199 | while (atomic_read(&start_count) != cpus) | |
200 | cpu_relax(); | |
201 | ||
b0e5c779 | 202 | check_tsc_warp(loop_timeout(smp_processor_id())); |
250c2277 TG |
203 | |
204 | /* | |
205 | * Ok, we are done: | |
206 | */ | |
207 | atomic_inc(&stop_count); | |
208 | ||
209 | /* | |
210 | * Wait for the source CPU to print stuff: | |
211 | */ | |
212 | while (atomic_read(&stop_count) != cpus) | |
213 | cpu_relax(); | |
214 | } |