arch/s390/kernel/vtime.c

   1 /*
   2  *    Virtual cpu timer based timer functions.
   3  *
   4  *    Copyright IBM Corp. 2004, 2012
   5  *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
   6  */
   7
   8 #include <linux/kernel_stat.h>
   9 #include <linux/notifier.h>
  10 #include <linux/kprobes.h>
  11 #include <linux/export.h>
  12 #include <linux/kernel.h>
  13 #include <linux/timex.h>
  14 #include <linux/types.h>
  15 #include <linux/time.h>
  16 #include <linux/cpu.h>
  17 #include <linux/smp.h>
  18
  19 #include <asm/irq_regs.h>
  20 #include <asm/cputime.h>
  21 #include <asm/vtimer.h>
  22 #include <asm/irq.h>
  23 #include "entry.h"
  24
  25 static void virt_timer_expire(void);
  26
  27 DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
  28
  29 static LIST_HEAD(virt_timer_list);
  30 static DEFINE_SPINLOCK(virt_timer_lock);
  31 static atomic64_t virt_timer_current;
  32 static atomic64_t virt_timer_elapsed;
  33
  34 static inline u64 get_vtimer(void)
  35 {
  36         u64 timer;
  37
  38         asm volatile("stpt %0" : "=m" (timer));
  39         return timer;
  40 }
  41
  42 static inline void set_vtimer(u64 expires)
  43 {
  44         u64 timer;
  45
  46         asm volatile(
  47                 "       stpt    %0\n"   /* Store current cpu timer value */
  48                 "       spt     %1"     /* Set new value imm. afterwards */
  49                 : "=m" (timer) : "m" (expires));
  50         S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
  51         S390_lowcore.last_update_timer = expires;
  52 }
  53
  54 static inline int virt_timer_forward(u64 elapsed)
  55 {
  56         BUG_ON(!irqs_disabled());
  57
  58         if (list_empty(&virt_timer_list))
  59                 return 0;
  60         elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
  61         return elapsed >= atomic64_read(&virt_timer_current);
  62 }
  63
  64 /*
  65  * Update process times based on virtual cpu times stored by entry.S
  66  * to the lowcore fields user_timer, system_timer & steal_clock.
  67  */
  68 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)
  69 {
  70         struct thread_info *ti = task_thread_info(tsk);
  71         u64 timer, clock, user, system, steal;
  72
  73         timer = S390_lowcore.last_update_timer;
  74         clock = S390_lowcore.last_update_clock;
  75         asm volatile(
  76                 "       stpt    %0\n"   /* Store current cpu timer value */
  77                 "       stck    %1"     /* Store current tod clock value */
  78                 : "=m" (S390_lowcore.last_update_timer),
  79                   "=m" (S390_lowcore.last_update_clock));
  80         S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
  81         S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
  82
  83         user = S390_lowcore.user_timer - ti->user_timer;
  84         S390_lowcore.steal_timer -= user;
  85         ti->user_timer = S390_lowcore.user_timer;
  86         account_user_time(tsk, user, user);
  87
  88         system = S390_lowcore.system_timer - ti->system_timer;
  89         S390_lowcore.steal_timer -= system;
  90         ti->system_timer = S390_lowcore.system_timer;
  91         account_system_time(tsk, hardirq_offset, system, system);
  92
  93         steal = S390_lowcore.steal_timer;
  94         if ((s64) steal > 0) {
  95                 S390_lowcore.steal_timer = 0;
  96                 account_steal_time(steal);
  97         }
  98
  99         return virt_timer_forward(user + system);
 100 }
 101
 102 void vtime_task_switch(struct task_struct *prev)
 103 {
 104         struct thread_info *ti;
 105
 106         do_account_vtime(prev, 0);
 107         ti = task_thread_info(prev);
 108         ti->user_timer = S390_lowcore.user_timer;
 109         ti->system_timer = S390_lowcore.system_timer;
 110         ti = task_thread_info(current);
 111         S390_lowcore.user_timer = ti->user_timer;
 112         S390_lowcore.system_timer = ti->system_timer;
 113 }
 114
 115 /*
 116  * In s390, accounting pending user time also implies
 117  * accounting system time in order to correctly compute
 118  * the stolen time accounting.
 119  */
 120 void vtime_account_user(struct task_struct *tsk)
 121 {
 122         if (do_account_vtime(tsk, HARDIRQ_OFFSET))
 123                 virt_timer_expire();
 124 }
 125
 126 /*
 127  * Update process times based on virtual cpu times stored by entry.S
 128  * to the lowcore fields user_timer, system_timer & steal_clock.
 129  */
 130 void vtime_account(struct task_struct *tsk)
 131 {
 132         struct thread_info *ti = task_thread_info(tsk);
 133         u64 timer, system;
 134
 135         WARN_ON_ONCE(!irqs_disabled());
 136
 137         timer = S390_lowcore.last_update_timer;
 138         S390_lowcore.last_update_timer = get_vtimer();
 139         S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
 140
 141         system = S390_lowcore.system_timer - ti->system_timer;
 142         S390_lowcore.steal_timer -= system;
 143         ti->system_timer = S390_lowcore.system_timer;
 144         account_system_time(tsk, 0, system, system);
 145
 146         virt_timer_forward(system);
 147 }
 148 EXPORT_SYMBOL_GPL(vtime_account);
 149
 150 void vtime_account_system(struct task_struct *tsk)
 151 __attribute__((alias("vtime_account")));
 152 EXPORT_SYMBOL_GPL(vtime_account_system);
 153
 154 void __kprobes vtime_stop_cpu(void)
 155 {
 156         struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
 157         unsigned long long idle_time;
 158         unsigned long psw_mask;
 159
 160         trace_hardirqs_on();
 161         /* Don't trace preempt off for idle. */
 162         stop_critical_timings();
 163
 164         /* Wait for external, I/O or machine check interrupt. */
 165         psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT |
 166                 PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
 167         idle->nohz_delay = 0;
 168
 169         /* Call the assembler magic in entry.S */
 170         psw_idle(idle, psw_mask);
 171
 172         /* Reenable preemption tracer. */
 173         start_critical_timings();
 174
 175         /* Account time spent with enabled wait psw loaded as idle time. */
 176         idle->sequence++;
 177         smp_wmb();
 178         idle_time = idle->clock_idle_exit - idle->clock_idle_enter;
 179         idle->clock_idle_enter = idle->clock_idle_exit = 0ULL;
 180         idle->idle_time += idle_time;
 181         idle->idle_count++;
 182         account_idle_time(idle_time);
 183         smp_wmb();
 184         idle->sequence++;
 185 }
 186
 187 cputime64_t s390_get_idle_time(int cpu)
 188 {
 189         struct s390_idle_data *idle = &per_cpu(s390_idle, cpu);
 190         unsigned long long now, idle_enter, idle_exit;
 191         unsigned int sequence;
 192
 193         do {
 194                 now = get_clock();
 195                 sequence = ACCESS_ONCE(idle->sequence);
 196                 idle_enter = ACCESS_ONCE(idle->clock_idle_enter);
 197                 idle_exit = ACCESS_ONCE(idle->clock_idle_exit);
 198         } while ((sequence & 1) || (idle->sequence != sequence));
 199         return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0;
 200 }
 201
 202 /*
 203  * Sorted add to a list. List is linear searched until first bigger
 204  * element is found.
 205  */
 206 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
 207 {
 208         struct vtimer_list *tmp;
 209
 210         list_for_each_entry(tmp, head, entry) {
 211                 if (tmp->expires > timer->expires) {
 212                         list_add_tail(&timer->entry, &tmp->entry);
 213                         return;
 214                 }
 215         }
 216         list_add_tail(&timer->entry, head);
 217 }
 218
 219 /*
 220  * Handler for expired virtual CPU timer.
 221  */
 222 static void virt_timer_expire(void)
 223 {
 224         struct vtimer_list *timer, *tmp;
 225         unsigned long elapsed;
 226         LIST_HEAD(cb_list);
 227
 228         /* walk timer list, fire all expired timers */
 229         spin_lock(&virt_timer_lock);
 230         elapsed = atomic64_read(&virt_timer_elapsed);
 231         list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
 232                 if (timer->expires < elapsed)
 233                         /* move expired timer to the callback queue */
 234                         list_move_tail(&timer->entry, &cb_list);
 235                 else
 236                         timer->expires -= elapsed;
 237         }
 238         if (!list_empty(&virt_timer_list)) {
 239                 timer = list_first_entry(&virt_timer_list,
 240                                          struct vtimer_list, entry);
 241                 atomic64_set(&virt_timer_current, timer->expires);
 242         }
 243         atomic64_sub(elapsed, &virt_timer_elapsed);
 244         spin_unlock(&virt_timer_lock);
 245
 246         /* Do callbacks and recharge periodic timers */
 247         list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
 248                 list_del_init(&timer->entry);
 249                 timer->function(timer->data);
 250                 if (timer->interval) {
 251                         /* Recharge interval timer */
 252                         timer->expires = timer->interval +
 253                                 atomic64_read(&virt_timer_elapsed);
 254                         spin_lock(&virt_timer_lock);
 255                         list_add_sorted(timer, &virt_timer_list);
 256                         spin_unlock(&virt_timer_lock);
 257                 }
 258         }
 259 }
 260
 261 void init_virt_timer(struct vtimer_list *timer)
 262 {
 263         timer->function = NULL;
 264         INIT_LIST_HEAD(&timer->entry);
 265 }
 266 EXPORT_SYMBOL(init_virt_timer);
 267
 268 static inline int vtimer_pending(struct vtimer_list *timer)
 269 {
 270         return !list_empty(&timer->entry);
 271 }
 272
 273 static void internal_add_vtimer(struct vtimer_list *timer)
 274 {
 275         if (list_empty(&virt_timer_list)) {
 276                 /* First timer, just program it. */
 277                 atomic64_set(&virt_timer_current, timer->expires);
 278                 atomic64_set(&virt_timer_elapsed, 0);
 279                 list_add(&timer->entry, &virt_timer_list);
 280         } else {
 281                 /* Update timer against current base. */
 282                 timer->expires += atomic64_read(&virt_timer_elapsed);
 283                 if (likely((s64) timer->expires <
 284                            (s64) atomic64_read(&virt_timer_current)))
 285                         /* The new timer expires before the current timer. */
 286                         atomic64_set(&virt_timer_current, timer->expires);
 287                 /* Insert new timer into the list. */
 288                 list_add_sorted(timer, &virt_timer_list);
 289         }
 290 }
 291
 292 static void __add_vtimer(struct vtimer_list *timer, int periodic)
 293 {
 294         unsigned long flags;
 295
 296         timer->interval = periodic ? timer->expires : 0;
 297         spin_lock_irqsave(&virt_timer_lock, flags);
 298         internal_add_vtimer(timer);
 299         spin_unlock_irqrestore(&virt_timer_lock, flags);
 300 }
 301
 302 /*
 303  * add_virt_timer - add an oneshot virtual CPU timer
 304  */
 305 void add_virt_timer(struct vtimer_list *timer)
 306 {
 307         __add_vtimer(timer, 0);
 308 }
 309 EXPORT_SYMBOL(add_virt_timer);
 310
 311 /*
 312  * add_virt_timer_int - add an interval virtual CPU timer
 313  */
 314 void add_virt_timer_periodic(struct vtimer_list *timer)
 315 {
 316         __add_vtimer(timer, 1);
 317 }
 318 EXPORT_SYMBOL(add_virt_timer_periodic);
 319
 320 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
 321 {
 322         unsigned long flags;
 323         int rc;
 324
 325         BUG_ON(!timer->function);
 326
 327         if (timer->expires == expires && vtimer_pending(timer))
 328                 return 1;
 329         spin_lock_irqsave(&virt_timer_lock, flags);
 330         rc = vtimer_pending(timer);
 331         if (rc)
 332                 list_del_init(&timer->entry);
 333         timer->interval = periodic ? expires : 0;
 334         timer->expires = expires;
 335         internal_add_vtimer(timer);
 336         spin_unlock_irqrestore(&virt_timer_lock, flags);
 337         return rc;
 338 }
 339
 340 /*
 341  * returns whether it has modified a pending timer (1) or not (0)
 342  */
 343 int mod_virt_timer(struct vtimer_list *timer, u64 expires)
 344 {
 345         return __mod_vtimer(timer, expires, 0);
 346 }
 347 EXPORT_SYMBOL(mod_virt_timer);
 348
 349 /*
 350  * returns whether it has modified a pending timer (1) or not (0)
 351  */
 352 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
 353 {
 354         return __mod_vtimer(timer, expires, 1);
 355 }
 356 EXPORT_SYMBOL(mod_virt_timer_periodic);
 357
 358 /*
 359  * Delete a virtual timer.
 360  *
 361  * returns whether the deleted timer was pending (1) or not (0)
 362  */
 363 int del_virt_timer(struct vtimer_list *timer)
 364 {
 365         unsigned long flags;
 366
 367         if (!vtimer_pending(timer))
 368                 return 0;
 369         spin_lock_irqsave(&virt_timer_lock, flags);
 370         list_del_init(&timer->entry);
 371         spin_unlock_irqrestore(&virt_timer_lock, flags);
 372         return 1;
 373 }
 374 EXPORT_SYMBOL(del_virt_timer);
 375
 376 /*
 377  * Start the virtual CPU timer on the current CPU.
 378  */
 379 void __cpuinit init_cpu_vtimer(void)
 380 {
 381         /* set initial cpu timer */
 382         set_vtimer(VTIMER_MAX_SLICE);
 383 }
 384
 385 static int __cpuinit s390_nohz_notify(struct notifier_block *self,
 386                                       unsigned long action, void *hcpu)
 387 {
 388         struct s390_idle_data *idle;
 389         long cpu = (long) hcpu;
 390
 391         idle = &per_cpu(s390_idle, cpu);
 392         switch (action & ~CPU_TASKS_FROZEN) {
 393         case CPU_DYING:
 394                 idle->nohz_delay = 0;
 395         default:
 396                 break;
 397         }
 398         return NOTIFY_OK;
 399 }
 400
 401 void __init vtime_init(void)
 402 {
 403         /* Enable cpu timer interrupts on the boot cpu. */
 404         init_cpu_vtimer();
 405         cpu_notifier(s390_nohz_notify, 0);
 406 }