drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/device.h>
  14 #include <linux/smp.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/clockchips.h>
  18 #include <linux/clocksource.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/of_address.h>
  22 #include <linux/io.h>
  23 #include <linux/slab.h>
  24 #include <linux/sched_clock.h>
  25 #include <linux/acpi.h>
  26
  27 #include <asm/arch_timer.h>
  28 #include <asm/virt.h>
  29
  30 #include <clocksource/arm_arch_timer.h>
  31
  32 #define CNTTIDR         0x08
  33 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  34
  35 #define CNTVCT_LO       0x08
  36 #define CNTVCT_HI       0x0c
  37 #define CNTFRQ          0x10
  38 #define CNTP_TVAL       0x28
  39 #define CNTP_CTL        0x2c
  40 #define CNTV_TVAL       0x38
  41 #define CNTV_CTL        0x3c
  42
  43 #define ARCH_CP15_TIMER BIT(0)
  44 #define ARCH_MEM_TIMER  BIT(1)
  45 static unsigned arch_timers_present __initdata;
  46
  47 static void __iomem *arch_counter_base;
  48
  49 struct arch_timer {
  50         void __iomem *base;
  51         struct clock_event_device evt;
  52 };
  53
  54 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  55
  56 static u32 arch_timer_rate;
  57
  58 enum ppi_nr {
  59         PHYS_SECURE_PPI,
  60         PHYS_NONSECURE_PPI,
  61         VIRT_PPI,
  62         HYP_PPI,
  63         MAX_TIMER_PPI
  64 };
  65
  66 static int arch_timer_ppi[MAX_TIMER_PPI];
  67
  68 static struct clock_event_device __percpu *arch_timer_evt;
  69
  70 static bool arch_timer_use_virtual = true;
  71 static bool arch_timer_c3stop;
  72 static bool arch_timer_mem_use_virtual;
  73
  74 /*
  75  * Architected system timer support.
  76  */
  77
  78 static __always_inline
  79 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
  80                           struct clock_event_device *clk)
  81 {
  82         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
  83                 struct arch_timer *timer = to_arch_timer(clk);
  84                 switch (reg) {
  85                 case ARCH_TIMER_REG_CTRL:
  86                         writel_relaxed(val, timer->base + CNTP_CTL);
  87                         break;
  88                 case ARCH_TIMER_REG_TVAL:
  89                         writel_relaxed(val, timer->base + CNTP_TVAL);
  90                         break;
  91                 }
  92         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
  93                 struct arch_timer *timer = to_arch_timer(clk);
  94                 switch (reg) {
  95                 case ARCH_TIMER_REG_CTRL:
  96                         writel_relaxed(val, timer->base + CNTV_CTL);
  97                         break;
  98                 case ARCH_TIMER_REG_TVAL:
  99                         writel_relaxed(val, timer->base + CNTV_TVAL);
 100                         break;
 101                 }
 102         } else {
 103                 arch_timer_reg_write_cp15(access, reg, val);
 104         }
 105 }
 106
 107 static __always_inline
 108 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 109                         struct clock_event_device *clk)
 110 {
 111         u32 val;
 112
 113         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 114                 struct arch_timer *timer = to_arch_timer(clk);
 115                 switch (reg) {
 116                 case ARCH_TIMER_REG_CTRL:
 117                         val = readl_relaxed(timer->base + CNTP_CTL);
 118                         break;
 119                 case ARCH_TIMER_REG_TVAL:
 120                         val = readl_relaxed(timer->base + CNTP_TVAL);
 121                         break;
 122                 }
 123         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 124                 struct arch_timer *timer = to_arch_timer(clk);
 125                 switch (reg) {
 126                 case ARCH_TIMER_REG_CTRL:
 127                         val = readl_relaxed(timer->base + CNTV_CTL);
 128                         break;
 129                 case ARCH_TIMER_REG_TVAL:
 130                         val = readl_relaxed(timer->base + CNTV_TVAL);
 131                         break;
 132                 }
 133         } else {
 134                 val = arch_timer_reg_read_cp15(access, reg);
 135         }
 136
 137         return val;
 138 }
 139
 140 static __always_inline irqreturn_t timer_handler(const int access,
 141                                         struct clock_event_device *evt)
 142 {
 143         unsigned long ctrl;
 144
 145         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 146         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 147                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 148                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 149                 evt->event_handler(evt);
 150                 return IRQ_HANDLED;
 151         }
 152
 153         return IRQ_NONE;
 154 }
 155
 156 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 157 {
 158         struct clock_event_device *evt = dev_id;
 159
 160         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 161 }
 162
 163 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 164 {
 165         struct clock_event_device *evt = dev_id;
 166
 167         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 168 }
 169
 170 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 171 {
 172         struct clock_event_device *evt = dev_id;
 173
 174         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 175 }
 176
 177 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 178 {
 179         struct clock_event_device *evt = dev_id;
 180
 181         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 182 }
 183
 184 static __always_inline int timer_shutdown(const int access,
 185                                           struct clock_event_device *clk)
 186 {
 187         unsigned long ctrl;
 188
 189         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 190         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 191         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 192
 193         return 0;
 194 }
 195
 196 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 197 {
 198         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 199 }
 200
 201 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 202 {
 203         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 204 }
 205
 206 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 207 {
 208         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 209 }
 210
 211 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 212 {
 213         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 214 }
 215
 216 static __always_inline void set_next_event(const int access, unsigned long evt,
 217                                            struct clock_event_device *clk)
 218 {
 219         unsigned long ctrl;
 220         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 221         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 222         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 223         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 224         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 225 }
 226
 227 static int arch_timer_set_next_event_virt(unsigned long evt,
 228                                           struct clock_event_device *clk)
 229 {
 230         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 231         return 0;
 232 }
 233
 234 static int arch_timer_set_next_event_phys(unsigned long evt,
 235                                           struct clock_event_device *clk)
 236 {
 237         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 238         return 0;
 239 }
 240
 241 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 242                                               struct clock_event_device *clk)
 243 {
 244         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 245         return 0;
 246 }
 247
 248 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 249                                               struct clock_event_device *clk)
 250 {
 251         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 252         return 0;
 253 }
 254
 255 static void __arch_timer_setup(unsigned type,
 256                                struct clock_event_device *clk)
 257 {
 258         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 259
 260         if (type == ARCH_CP15_TIMER) {
 261                 if (arch_timer_c3stop)
 262                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 263                 clk->name = "arch_sys_timer";
 264                 clk->rating = 450;
 265                 clk->cpumask = cpumask_of(smp_processor_id());
 266                 if (arch_timer_use_virtual) {
 267                         clk->irq = arch_timer_ppi[VIRT_PPI];
 268                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 269                         clk->set_next_event = arch_timer_set_next_event_virt;
 270                 } else {
 271                         clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
 272                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 273                         clk->set_next_event = arch_timer_set_next_event_phys;
 274                 }
 275         } else {
 276                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 277                 clk->name = "arch_mem_timer";
 278                 clk->rating = 400;
 279                 clk->cpumask = cpu_all_mask;
 280                 if (arch_timer_mem_use_virtual) {
 281                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 282                         clk->set_next_event =
 283                                 arch_timer_set_next_event_virt_mem;
 284                 } else {
 285                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 286                         clk->set_next_event =
 287                                 arch_timer_set_next_event_phys_mem;
 288                 }
 289         }
 290
 291         clk->set_state_shutdown(clk);
 292
 293         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 294 }
 295
 296 static void arch_timer_evtstrm_enable(int divider)
 297 {
 298         u32 cntkctl = arch_timer_get_cntkctl();
 299
 300         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 301         /* Set the divider and enable virtual event stream */
 302         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 303                         | ARCH_TIMER_VIRT_EVT_EN;
 304         arch_timer_set_cntkctl(cntkctl);
 305         elf_hwcap |= HWCAP_EVTSTRM;
 306 #ifdef CONFIG_COMPAT
 307         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 308 #endif
 309 }
 310
 311 static void arch_timer_configure_evtstream(void)
 312 {
 313         int evt_stream_div, pos;
 314
 315         /* Find the closest power of two to the divisor */
 316         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 317         pos = fls(evt_stream_div);
 318         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 319                 pos--;
 320         /* enable event stream */
 321         arch_timer_evtstrm_enable(min(pos, 15));
 322 }
 323
 324 static void arch_counter_set_user_access(void)
 325 {
 326         u32 cntkctl = arch_timer_get_cntkctl();
 327
 328         /* Disable user access to the timers and the physical counter */
 329         /* Also disable virtual event stream */
 330         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 331                         | ARCH_TIMER_USR_VT_ACCESS_EN
 332                         | ARCH_TIMER_VIRT_EVT_EN
 333                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 334
 335         /* Enable user access to the virtual counter */
 336         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 337
 338         arch_timer_set_cntkctl(cntkctl);
 339 }
 340
 341 static int arch_timer_setup(struct clock_event_device *clk)
 342 {
 343         __arch_timer_setup(ARCH_CP15_TIMER, clk);
 344
 345         if (arch_timer_use_virtual)
 346                 enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
 347         else {
 348                 enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
 349                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 350                         enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
 351         }
 352
 353         arch_counter_set_user_access();
 354         if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM))
 355                 arch_timer_configure_evtstream();
 356
 357         return 0;
 358 }
 359
 360 static void
 361 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
 362 {
 363         /* Who has more than one independent system counter? */
 364         if (arch_timer_rate)
 365                 return;
 366
 367         /*
 368          * Try to determine the frequency from the device tree or CNTFRQ,
 369          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
 370          */
 371         if (!acpi_disabled ||
 372             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
 373                 if (cntbase)
 374                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
 375                 else
 376                         arch_timer_rate = arch_timer_get_cntfrq();
 377         }
 378
 379         /* Check the timer frequency. */
 380         if (arch_timer_rate == 0)
 381                 pr_warn("Architected timer frequency not available\n");
 382 }
 383
 384 static void arch_timer_banner(unsigned type)
 385 {
 386         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 387                      type & ARCH_CP15_TIMER ? "cp15" : "",
 388                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
 389                      type & ARCH_MEM_TIMER ? "mmio" : "",
 390                      (unsigned long)arch_timer_rate / 1000000,
 391                      (unsigned long)(arch_timer_rate / 10000) % 100,
 392                      type & ARCH_CP15_TIMER ?
 393                         arch_timer_use_virtual ? "virt" : "phys" :
 394                         "",
 395                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
 396                      type & ARCH_MEM_TIMER ?
 397                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 398                         "");
 399 }
 400
 401 u32 arch_timer_get_rate(void)
 402 {
 403         return arch_timer_rate;
 404 }
 405
 406 static u64 arch_counter_get_cntvct_mem(void)
 407 {
 408         u32 vct_lo, vct_hi, tmp_hi;
 409
 410         do {
 411                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 412                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 413                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 414         } while (vct_hi != tmp_hi);
 415
 416         return ((u64) vct_hi << 32) | vct_lo;
 417 }
 418
 419 /*
 420  * Default to cp15 based access because arm64 uses this function for
 421  * sched_clock() before DT is probed and the cp15 method is guaranteed
 422  * to exist on arm64. arm doesn't use this before DT is probed so even
 423  * if we don't have the cp15 accessors we won't have a problem.
 424  */
 425 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 426
 427 static cycle_t arch_counter_read(struct clocksource *cs)
 428 {
 429         return arch_timer_read_counter();
 430 }
 431
 432 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
 433 {
 434         return arch_timer_read_counter();
 435 }
 436
 437 static struct clocksource clocksource_counter = {
 438         .name   = "arch_sys_counter",
 439         .rating = 400,
 440         .read   = arch_counter_read,
 441         .mask   = CLOCKSOURCE_MASK(56),
 442         .flags  = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
 443 };
 444
 445 static struct cyclecounter cyclecounter = {
 446         .read   = arch_counter_read_cc,
 447         .mask   = CLOCKSOURCE_MASK(56),
 448 };
 449
 450 static struct timecounter timecounter;
 451
 452 struct timecounter *arch_timer_get_timecounter(void)
 453 {
 454         return &timecounter;
 455 }
 456
 457 static void __init arch_counter_register(unsigned type)
 458 {
 459         u64 start_count;
 460
 461         /* Register the CP15 based counter if we have one */
 462         if (type & ARCH_CP15_TIMER) {
 463                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_use_virtual)
 464                         arch_timer_read_counter = arch_counter_get_cntvct;
 465                 else
 466                         arch_timer_read_counter = arch_counter_get_cntpct;
 467         } else {
 468                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
 469
 470                 /* If the clocksource name is "arch_sys_counter" the
 471                  * VDSO will attempt to read the CP15-based counter.
 472                  * Ensure this does not happen when CP15-based
 473                  * counter is not available.
 474                  */
 475                 clocksource_counter.name = "arch_mem_counter";
 476         }
 477
 478         start_count = arch_timer_read_counter();
 479         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 480         cyclecounter.mult = clocksource_counter.mult;
 481         cyclecounter.shift = clocksource_counter.shift;
 482         timecounter_init(&timecounter, &cyclecounter, start_count);
 483
 484         /* 56 bits minimum, so we assume worst case rollover */
 485         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 486 }
 487
 488 static void arch_timer_stop(struct clock_event_device *clk)
 489 {
 490         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 491                  clk->irq, smp_processor_id());
 492
 493         if (arch_timer_use_virtual)
 494                 disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
 495         else {
 496                 disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
 497                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 498                         disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 499         }
 500
 501         clk->set_state_shutdown(clk);
 502 }
 503
 504 static int arch_timer_cpu_notify(struct notifier_block *self,
 505                                            unsigned long action, void *hcpu)
 506 {
 507         /*
 508          * Grab cpu pointer in each case to avoid spurious
 509          * preemptible warnings
 510          */
 511         switch (action & ~CPU_TASKS_FROZEN) {
 512         case CPU_STARTING:
 513                 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 514                 break;
 515         case CPU_DYING:
 516                 arch_timer_stop(this_cpu_ptr(arch_timer_evt));
 517                 break;
 518         }
 519
 520         return NOTIFY_OK;
 521 }
 522
 523 static struct notifier_block arch_timer_cpu_nb = {
 524         .notifier_call = arch_timer_cpu_notify,
 525 };
 526
 527 #ifdef CONFIG_CPU_PM
 528 static unsigned int saved_cntkctl;
 529 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 530                                     unsigned long action, void *hcpu)
 531 {
 532         if (action == CPU_PM_ENTER)
 533                 saved_cntkctl = arch_timer_get_cntkctl();
 534         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 535                 arch_timer_set_cntkctl(saved_cntkctl);
 536         return NOTIFY_OK;
 537 }
 538
 539 static struct notifier_block arch_timer_cpu_pm_notifier = {
 540         .notifier_call = arch_timer_cpu_pm_notify,
 541 };
 542
 543 static int __init arch_timer_cpu_pm_init(void)
 544 {
 545         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 546 }
 547 #else
 548 static int __init arch_timer_cpu_pm_init(void)
 549 {
 550         return 0;
 551 }
 552 #endif
 553
 554 static int __init arch_timer_register(void)
 555 {
 556         int err;
 557         int ppi;
 558
 559         arch_timer_evt = alloc_percpu(struct clock_event_device);
 560         if (!arch_timer_evt) {
 561                 err = -ENOMEM;
 562                 goto out;
 563         }
 564
 565         if (arch_timer_use_virtual) {
 566                 ppi = arch_timer_ppi[VIRT_PPI];
 567                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 568                                          "arch_timer", arch_timer_evt);
 569         } else {
 570                 ppi = arch_timer_ppi[PHYS_SECURE_PPI];
 571                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 572                                          "arch_timer", arch_timer_evt);
 573                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 574                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 575                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 576                                                  "arch_timer", arch_timer_evt);
 577                         if (err)
 578                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 579                                                 arch_timer_evt);
 580                 }
 581         }
 582
 583         if (err) {
 584                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 585                        ppi, err);
 586                 goto out_free;
 587         }
 588
 589         err = register_cpu_notifier(&arch_timer_cpu_nb);
 590         if (err)
 591                 goto out_free_irq;
 592
 593         err = arch_timer_cpu_pm_init();
 594         if (err)
 595                 goto out_unreg_notify;
 596
 597         /* Immediately configure the timer on the boot CPU */
 598         arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 599
 600         return 0;
 601
 602 out_unreg_notify:
 603         unregister_cpu_notifier(&arch_timer_cpu_nb);
 604 out_free_irq:
 605         if (arch_timer_use_virtual)
 606                 free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
 607         else {
 608                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 609                                 arch_timer_evt);
 610                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 611                         free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 612                                         arch_timer_evt);
 613         }
 614
 615 out_free:
 616         free_percpu(arch_timer_evt);
 617 out:
 618         return err;
 619 }
 620
 621 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
 622 {
 623         int ret;
 624         irq_handler_t func;
 625         struct arch_timer *t;
 626
 627         t = kzalloc(sizeof(*t), GFP_KERNEL);
 628         if (!t)
 629                 return -ENOMEM;
 630
 631         t->base = base;
 632         t->evt.irq = irq;
 633         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
 634
 635         if (arch_timer_mem_use_virtual)
 636                 func = arch_timer_handler_virt_mem;
 637         else
 638                 func = arch_timer_handler_phys_mem;
 639
 640         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
 641         if (ret) {
 642                 pr_err("arch_timer: Failed to request mem timer irq\n");
 643                 kfree(t);
 644         }
 645
 646         return ret;
 647 }
 648
 649 static const struct of_device_id arch_timer_of_match[] __initconst = {
 650         { .compatible   = "arm,armv7-timer",    },
 651         { .compatible   = "arm,armv8-timer",    },
 652         {},
 653 };
 654
 655 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
 656         { .compatible   = "arm,armv7-timer-mem", },
 657         {},
 658 };
 659
 660 static bool __init
 661 arch_timer_needs_probing(int type, const struct of_device_id *matches)
 662 {
 663         struct device_node *dn;
 664         bool needs_probing = false;
 665
 666         dn = of_find_matching_node(NULL, matches);
 667         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
 668                 needs_probing = true;
 669         of_node_put(dn);
 670
 671         return needs_probing;
 672 }
 673
 674 static void __init arch_timer_common_init(void)
 675 {
 676         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
 677
 678         /* Wait until both nodes are probed if we have two timers */
 679         if ((arch_timers_present & mask) != mask) {
 680                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
 681                         return;
 682                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
 683                         return;
 684         }
 685
 686         arch_timer_banner(arch_timers_present);
 687         arch_counter_register(arch_timers_present);
 688         arch_timer_arch_init();
 689 }
 690
 691 static void __init arch_timer_init(void)
 692 {
 693         /*
 694          * If HYP mode is available, we know that the physical timer
 695          * has been configured to be accessible from PL1. Use it, so
 696          * that a guest can use the virtual timer instead.
 697          *
 698          * If no interrupt provided for virtual timer, we'll have to
 699          * stick to the physical timer. It'd better be accessible...
 700          */
 701         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 702                 arch_timer_use_virtual = false;
 703
 704                 if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
 705                     !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 706                         pr_warn("arch_timer: No interrupt available, giving up\n");
 707                         return;
 708                 }
 709         }
 710
 711         arch_timer_register();
 712         arch_timer_common_init();
 713 }
 714
 715 static void __init arch_timer_of_init(struct device_node *np)
 716 {
 717         int i;
 718
 719         if (arch_timers_present & ARCH_CP15_TIMER) {
 720                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 721                 return;
 722         }
 723
 724         arch_timers_present |= ARCH_CP15_TIMER;
 725         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 726                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 727
 728         arch_timer_detect_rate(NULL, np);
 729
 730         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 731
 732         /*
 733          * If we cannot rely on firmware initializing the timer registers then
 734          * we should use the physical timers instead.
 735          */
 736         if (IS_ENABLED(CONFIG_ARM) &&
 737             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
 738                         arch_timer_use_virtual = false;
 739
 740         arch_timer_init();
 741 }
 742 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
 743 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
 744
 745 static void __init arch_timer_mem_init(struct device_node *np)
 746 {
 747         struct device_node *frame, *best_frame = NULL;
 748         void __iomem *cntctlbase, *base;
 749         unsigned int irq;
 750         u32 cnttidr;
 751
 752         arch_timers_present |= ARCH_MEM_TIMER;
 753         cntctlbase = of_iomap(np, 0);
 754         if (!cntctlbase) {
 755                 pr_err("arch_timer: Can't find CNTCTLBase\n");
 756                 return;
 757         }
 758
 759         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
 760         iounmap(cntctlbase);
 761
 762         /*
 763          * Try to find a virtual capable frame. Otherwise fall back to a
 764          * physical capable frame.
 765          */
 766         for_each_available_child_of_node(np, frame) {
 767                 int n;
 768
 769                 if (of_property_read_u32(frame, "frame-number", &n)) {
 770                         pr_err("arch_timer: Missing frame-number\n");
 771                         of_node_put(best_frame);
 772                         of_node_put(frame);
 773                         return;
 774                 }
 775
 776                 if (cnttidr & CNTTIDR_VIRT(n)) {
 777                         of_node_put(best_frame);
 778                         best_frame = frame;
 779                         arch_timer_mem_use_virtual = true;
 780                         break;
 781                 }
 782                 of_node_put(best_frame);
 783                 best_frame = of_node_get(frame);
 784         }
 785
 786         base = arch_counter_base = of_iomap(best_frame, 0);
 787         if (!base) {
 788                 pr_err("arch_timer: Can't map frame's registers\n");
 789                 of_node_put(best_frame);
 790                 return;
 791         }
 792
 793         if (arch_timer_mem_use_virtual)
 794                 irq = irq_of_parse_and_map(best_frame, 1);
 795         else
 796                 irq = irq_of_parse_and_map(best_frame, 0);
 797         of_node_put(best_frame);
 798         if (!irq) {
 799                 pr_err("arch_timer: Frame missing %s irq",
 800                        arch_timer_mem_use_virtual ? "virt" : "phys");
 801                 return;
 802         }
 803
 804         arch_timer_detect_rate(base, np);
 805         arch_timer_mem_register(base, irq);
 806         arch_timer_common_init();
 807 }
 808 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
 809                        arch_timer_mem_init);
 810
 811 #ifdef CONFIG_ACPI
 812 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
 813 {
 814         int trigger, polarity;
 815
 816         if (!interrupt)
 817                 return 0;
 818
 819         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
 820                         : ACPI_LEVEL_SENSITIVE;
 821
 822         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
 823                         : ACPI_ACTIVE_HIGH;
 824
 825         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
 826 }
 827
 828 /* Initialize per-processor generic timer */
 829 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
 830 {
 831         struct acpi_table_gtdt *gtdt;
 832
 833         if (arch_timers_present & ARCH_CP15_TIMER) {
 834                 pr_warn("arch_timer: already initialized, skipping\n");
 835                 return -EINVAL;
 836         }
 837
 838         gtdt = container_of(table, struct acpi_table_gtdt, header);
 839
 840         arch_timers_present |= ARCH_CP15_TIMER;
 841
 842         arch_timer_ppi[PHYS_SECURE_PPI] =
 843                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
 844                 gtdt->secure_el1_flags);
 845
 846         arch_timer_ppi[PHYS_NONSECURE_PPI] =
 847                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
 848                 gtdt->non_secure_el1_flags);
 849
 850         arch_timer_ppi[VIRT_PPI] =
 851                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
 852                 gtdt->virtual_timer_flags);
 853
 854         arch_timer_ppi[HYP_PPI] =
 855                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
 856                 gtdt->non_secure_el2_flags);
 857
 858         /* Get the frequency from CNTFRQ */
 859         arch_timer_detect_rate(NULL, NULL);
 860
 861         /* Always-on capability */
 862         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
 863
 864         arch_timer_init();
 865         return 0;
 866 }
 867
 868 /* Initialize all the generic timers presented in GTDT */
 869 void __init acpi_generic_timer_init(void)
 870 {
 871         if (acpi_disabled)
 872                 return;
 873
 874         acpi_table_parse(ACPI_SIG_GTDT, arch_timer_acpi_init);
 875 }
 876 #endif