drivers/crypto/caam/jr.c

   1 /*
   2  * CAAM/SEC 4.x transport/backend driver
   3  * JobR backend functionality
   4  *
   5  * Copyright 2008-2012 Freescale Semiconductor, Inc.
   6  */
   7
   8 #include "compat.h"
   9 #include "regs.h"
  10 #include "jr.h"
  11 #include "desc.h"
  12 #include "intern.h"
  13
  14 /* Main per-ring interrupt handler */
  15 static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
  16 {
  17         struct device *dev = st_dev;
  18         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
  19         u32 irqstate;
  20
  21         /*
  22          * Check the output ring for ready responses, kick
  23          * tasklet if jobs done.
  24          */
  25         irqstate = rd_reg32(&jrp->rregs->jrintstatus);
  26         if (!irqstate)
  27                 return IRQ_NONE;
  28
  29         /*
  30          * If JobR error, we got more development work to do
  31          * Flag a bug now, but we really need to shut down and
  32          * restart the queue (and fix code).
  33          */
  34         if (irqstate & JRINT_JR_ERROR) {
  35                 dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
  36                 BUG();
  37         }
  38
  39         /* mask valid interrupts */
  40         setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
  41
  42         /* Have valid interrupt at this point, just ACK and trigger */
  43         wr_reg32(&jrp->rregs->jrintstatus, irqstate);
  44
  45         preempt_disable();
  46         tasklet_schedule(&jrp->irqtask);
  47         preempt_enable();
  48
  49         return IRQ_HANDLED;
  50 }
  51
  52 /* Deferred service handler, run as interrupt-fired tasklet */
  53 static void caam_jr_dequeue(unsigned long devarg)
  54 {
  55         int hw_idx, sw_idx, i, head, tail;
  56         struct device *dev = (struct device *)devarg;
  57         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
  58         void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
  59         u32 *userdesc, userstatus;
  60         void *userarg;
  61
  62         while (rd_reg32(&jrp->rregs->outring_used)) {
  63
  64                 head = ACCESS_ONCE(jrp->head);
  65
  66                 spin_lock(&jrp->outlock);
  67
  68                 sw_idx = tail = jrp->tail;
  69                 hw_idx = jrp->out_ring_read_index;
  70
  71                 for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
  72                         sw_idx = (tail + i) & (JOBR_DEPTH - 1);
  73
  74                         smp_read_barrier_depends();
  75
  76                         if (jrp->outring[hw_idx].desc ==
  77                             jrp->entinfo[sw_idx].desc_addr_dma)
  78                                 break; /* found */
  79                 }
  80                 /* we should never fail to find a matching descriptor */
  81                 BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
  82
  83                 /* Unmap just-run descriptor so we can post-process */
  84                 dma_unmap_single(dev, jrp->outring[hw_idx].desc,
  85                                  jrp->entinfo[sw_idx].desc_size,
  86                                  DMA_TO_DEVICE);
  87
  88                 /* mark completed, avoid matching on a recycled desc addr */
  89                 jrp->entinfo[sw_idx].desc_addr_dma = 0;
  90
  91                 /* Stash callback params for use outside of lock */
  92                 usercall = jrp->entinfo[sw_idx].callbk;
  93                 userarg = jrp->entinfo[sw_idx].cbkarg;
  94                 userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
  95                 userstatus = jrp->outring[hw_idx].jrstatus;
  96
  97                 /* set done */
  98                 wr_reg32(&jrp->rregs->outring_rmvd, 1);
  99
 100                 jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
 101                                            (JOBR_DEPTH - 1);
 102
 103                 /*
 104                  * if this job completed out-of-order, do not increment
 105                  * the tail.  Otherwise, increment tail by 1 plus the
 106                  * number of subsequent jobs already completed out-of-order
 107                  */
 108                 if (sw_idx == tail) {
 109                         do {
 110                                 tail = (tail + 1) & (JOBR_DEPTH - 1);
 111                                 smp_read_barrier_depends();
 112                         } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
 113                                  jrp->entinfo[tail].desc_addr_dma == 0);
 114
 115                         jrp->tail = tail;
 116                 }
 117
 118                 spin_unlock(&jrp->outlock);
 119
 120                 /* Finally, execute user's callback */
 121                 usercall(dev, userdesc, userstatus, userarg);
 122         }
 123
 124         /* reenable / unmask IRQs */
 125         clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 126 }
 127
 128 /**
 129  * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
 130  * -EBUSY if the queue is full, -EIO if it cannot map the caller's
 131  * descriptor.
 132  * @dev:  device of the job ring to be used. This device should have
 133  *        been assigned prior by caam_jr_register().
 134  * @desc: points to a job descriptor that execute our request. All
 135  *        descriptors (and all referenced data) must be in a DMAable
 136  *        region, and all data references must be physical addresses
 137  *        accessible to CAAM (i.e. within a PAMU window granted
 138  *        to it).
 139  * @cbk:  pointer to a callback function to be invoked upon completion
 140  *        of this request. This has the form:
 141  *        callback(struct device *dev, u32 *desc, u32 stat, void *arg)
 142  *        where:
 143  *        @dev:    contains the job ring device that processed this
 144  *                 response.
 145  *        @desc:   descriptor that initiated the request, same as
 146  *                 "desc" being argued to caam_jr_enqueue().
 147  *        @status: untranslated status received from CAAM. See the
 148  *                 reference manual for a detailed description of
 149  *                 error meaning, or see the JRSTA definitions in the
 150  *                 register header file
 151  *        @areq:   optional pointer to an argument passed with the
 152  *                 original request
 153  * @areq: optional pointer to a user argument for use at callback
 154  *        time.
 155  **/
 156 int caam_jr_enqueue(struct device *dev, u32 *desc,
 157                     void (*cbk)(struct device *dev, u32 *desc,
 158                                 u32 status, void *areq),
 159                     void *areq)
 160 {
 161         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 162         struct caam_jrentry_info *head_entry;
 163         int head, tail, desc_size;
 164         dma_addr_t desc_dma;
 165
 166         desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32);
 167         desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
 168         if (dma_mapping_error(dev, desc_dma)) {
 169                 dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
 170                 return -EIO;
 171         }
 172
 173         spin_lock_bh(&jrp->inplock);
 174
 175         head = jrp->head;
 176         tail = ACCESS_ONCE(jrp->tail);
 177
 178         if (!rd_reg32(&jrp->rregs->inpring_avail) ||
 179             CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
 180                 spin_unlock_bh(&jrp->inplock);
 181                 dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
 182                 return -EBUSY;
 183         }
 184
 185         head_entry = &jrp->entinfo[head];
 186         head_entry->desc_addr_virt = desc;
 187         head_entry->desc_size = desc_size;
 188         head_entry->callbk = (void *)cbk;
 189         head_entry->cbkarg = areq;
 190         head_entry->desc_addr_dma = desc_dma;
 191
 192         jrp->inpring[jrp->inp_ring_write_index] = desc_dma;
 193
 194         smp_wmb();
 195
 196         jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
 197                                     (JOBR_DEPTH - 1);
 198         jrp->head = (head + 1) & (JOBR_DEPTH - 1);
 199
 200         wr_reg32(&jrp->rregs->inpring_jobadd, 1);
 201
 202         spin_unlock_bh(&jrp->inplock);
 203
 204         return 0;
 205 }
 206 EXPORT_SYMBOL(caam_jr_enqueue);
 207
 208 static int caam_reset_hw_jr(struct device *dev)
 209 {
 210         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 211         unsigned int timeout = 100000;
 212
 213         /*
 214          * mask interrupts since we are going to poll
 215          * for reset completion status
 216          */
 217         setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 218
 219         /* initiate flush (required prior to reset) */
 220         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
 221         while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
 222                 JRINT_ERR_HALT_INPROGRESS) && --timeout)
 223                 cpu_relax();
 224
 225         if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
 226             JRINT_ERR_HALT_COMPLETE || timeout == 0) {
 227                 dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
 228                 return -EIO;
 229         }
 230
 231         /* initiate reset */
 232         timeout = 100000;
 233         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
 234         while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
 235                 cpu_relax();
 236
 237         if (timeout == 0) {
 238                 dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
 239                 return -EIO;
 240         }
 241
 242         /* unmask interrupts */
 243         clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 244
 245         return 0;
 246 }
 247
 248 /*
 249  * Init JobR independent of platform property detection
 250  */
 251 static int caam_jr_init(struct device *dev)
 252 {
 253         struct caam_drv_private_jr *jrp;
 254         dma_addr_t inpbusaddr, outbusaddr;
 255         int i, error;
 256
 257         jrp = dev_get_drvdata(dev);
 258
 259         tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
 260
 261         /* Connect job ring interrupt handler. */
 262         error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
 263                             "caam-jobr", dev);
 264         if (error) {
 265                 dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
 266                         jrp->ridx, jrp->irq);
 267                 irq_dispose_mapping(jrp->irq);
 268                 jrp->irq = 0;
 269                 return -EINVAL;
 270         }
 271
 272         error = caam_reset_hw_jr(dev);
 273         if (error)
 274                 return error;
 275
 276         jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
 277                                           &inpbusaddr, GFP_KERNEL);
 278
 279         jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) *
 280                                           JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
 281
 282         jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
 283                                GFP_KERNEL);
 284
 285         if ((jrp->inpring == NULL) || (jrp->outring == NULL) ||
 286             (jrp->entinfo == NULL)) {
 287                 dev_err(dev, "can't allocate job rings for %d\n",
 288                         jrp->ridx);
 289                 return -ENOMEM;
 290         }
 291
 292         for (i = 0; i < JOBR_DEPTH; i++)
 293                 jrp->entinfo[i].desc_addr_dma = !0;
 294
 295         /* Setup rings */
 296         jrp->inp_ring_write_index = 0;
 297         jrp->out_ring_read_index = 0;
 298         jrp->head = 0;
 299         jrp->tail = 0;
 300
 301         wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
 302         wr_reg64(&jrp->rregs->outring_base, outbusaddr);
 303         wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
 304         wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
 305
 306         jrp->ringsize = JOBR_DEPTH;
 307
 308         spin_lock_init(&jrp->inplock);
 309         spin_lock_init(&jrp->outlock);
 310
 311         /* Select interrupt coalescing parameters */
 312         setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC |
 313                   (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
 314                   (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * Shutdown JobR independent of platform property code
 321  */
 322 int caam_jr_shutdown(struct device *dev)
 323 {
 324         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 325         dma_addr_t inpbusaddr, outbusaddr;
 326         int ret;
 327
 328         ret = caam_reset_hw_jr(dev);
 329
 330         tasklet_kill(&jrp->irqtask);
 331
 332         /* Release interrupt */
 333         free_irq(jrp->irq, dev);
 334
 335         /* Free rings */
 336         inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
 337         outbusaddr = rd_reg64(&jrp->rregs->outring_base);
 338         dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
 339                           jrp->inpring, inpbusaddr);
 340         dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
 341                           jrp->outring, outbusaddr);
 342         kfree(jrp->entinfo);
 343         of_device_unregister(jrp->jr_pdev);
 344
 345         return ret;
 346 }
 347
 348 /*
 349  * Probe routine for each detected JobR subsystem. It assumes that
 350  * property detection was picked up externally.
 351  */
 352 int caam_jr_probe(struct platform_device *pdev, struct device_node *np,
 353                   int ring)
 354 {
 355         struct device *ctrldev, *jrdev;
 356         struct platform_device *jr_pdev;
 357         struct caam_drv_private *ctrlpriv;
 358         struct caam_drv_private_jr *jrpriv;
 359         u32 *jroffset;
 360         int error;
 361
 362         ctrldev = &pdev->dev;
 363         ctrlpriv = dev_get_drvdata(ctrldev);
 364
 365         jrpriv = kmalloc(sizeof(struct caam_drv_private_jr),
 366                          GFP_KERNEL);
 367         if (jrpriv == NULL) {
 368                 dev_err(ctrldev, "can't alloc private mem for job ring %d\n",
 369                         ring);
 370                 return -ENOMEM;
 371         }
 372         jrpriv->parentdev = ctrldev; /* point back to parent */
 373         jrpriv->ridx = ring; /* save ring identity relative to detection */
 374
 375         /*
 376          * Derive a pointer to the detected JobRs regs
 377          * Driver has already iomapped the entire space, we just
 378          * need to add in the offset to this JobR. Don't know if I
 379          * like this long-term, but it'll run
 380          */
 381         jroffset = (u32 *)of_get_property(np, "reg", NULL);
 382         jrpriv->rregs = (struct caam_job_ring __iomem *)((void *)ctrlpriv->ctrl
 383                                                          + *jroffset);
 384
 385         /* Build a local dev for each detected queue */
 386         jr_pdev = of_platform_device_create(np, NULL, ctrldev);
 387         if (jr_pdev == NULL) {
 388                 kfree(jrpriv);
 389                 return -EINVAL;
 390         }
 391
 392         jrpriv->jr_pdev = jr_pdev;
 393         jrdev = &jr_pdev->dev;
 394         dev_set_drvdata(jrdev, jrpriv);
 395         ctrlpriv->jrdev[ring] = jrdev;
 396
 397         if (sizeof(dma_addr_t) == sizeof(u64))
 398                 if (of_device_is_compatible(np, "fsl,sec-v5.0-job-ring"))
 399                         dma_set_mask(jrdev, DMA_BIT_MASK(40));
 400                 else
 401                         dma_set_mask(jrdev, DMA_BIT_MASK(36));
 402         else
 403                 dma_set_mask(jrdev, DMA_BIT_MASK(32));
 404
 405         /* Identify the interrupt */
 406         jrpriv->irq = of_irq_to_resource(np, 0, NULL);
 407
 408         /* Now do the platform independent part */
 409         error = caam_jr_init(jrdev); /* now turn on hardware */
 410         if (error) {
 411                 of_device_unregister(jr_pdev);
 412                 kfree(jrpriv);
 413                 return error;
 414         }
 415
 416         return error;
 417 }