drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c

   1 /*
   2  * cxgb4_uld.c:Chelsio Upper Layer Driver Interface for T4/T5/T6 SGE management
   3  *
   4  * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  *  Written by: Atul Gupta (atul.gupta@chelsio.com)
  35  *  Written by: Hariprasad Shenai (hariprasad@chelsio.com)
  36  */
  37
  38 #include <linux/kernel.h>
  39 #include <linux/module.h>
  40 #include <linux/errno.h>
  41 #include <linux/types.h>
  42 #include <linux/debugfs.h>
  43 #include <linux/export.h>
  44 #include <linux/list.h>
  45 #include <linux/skbuff.h>
  46 #include <linux/pci.h>
  47
  48 #include "cxgb4.h"
  49 #include "cxgb4_uld.h"
  50 #include "t4_regs.h"
  51 #include "t4fw_api.h"
  52 #include "t4_msg.h"
  53
  54 #define for_each_uldrxq(m, i) for (i = 0; i < ((m)->nrxq + (m)->nciq); i++)
  55
  56 static int get_msix_idx_from_bmap(struct adapter *adap)
  57 {
  58         struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
  59         unsigned long flags;
  60         unsigned int msix_idx;
  61
  62         spin_lock_irqsave(&bmap->lock, flags);
  63         msix_idx = find_first_zero_bit(bmap->msix_bmap, bmap->mapsize);
  64         if (msix_idx < bmap->mapsize) {
  65                 __set_bit(msix_idx, bmap->msix_bmap);
  66         } else {
  67                 spin_unlock_irqrestore(&bmap->lock, flags);
  68                 return -ENOSPC;
  69         }
  70
  71         spin_unlock_irqrestore(&bmap->lock, flags);
  72         return msix_idx;
  73 }
  74
  75 static void free_msix_idx_in_bmap(struct adapter *adap, unsigned int msix_idx)
  76 {
  77         struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
  78         unsigned long flags;
  79
  80         spin_lock_irqsave(&bmap->lock, flags);
  81          __clear_bit(msix_idx, bmap->msix_bmap);
  82         spin_unlock_irqrestore(&bmap->lock, flags);
  83 }
  84
  85 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
  86                          const struct pkt_gl *gl)
  87 {
  88         struct adapter *adap = q->adap;
  89         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
  90         int ret;
  91
  92         /* FW can send CPLs encapsulated in a CPL_FW4_MSG */
  93         if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
  94             ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
  95                 rsp += 2;
  96
  97         if (q->flush_handler)
  98                 ret = adap->uld[q->uld].lro_rx_handler(adap->uld[q->uld].handle,
  99                                 rsp, gl, &q->lro_mgr,
 100                                 &q->napi);
 101         else
 102                 ret = adap->uld[q->uld].rx_handler(adap->uld[q->uld].handle,
 103                                 rsp, gl);
 104
 105         if (ret) {
 106                 rxq->stats.nomem++;
 107                 return -1;
 108         }
 109
 110         if (!gl)
 111                 rxq->stats.imm++;
 112         else if (gl == CXGB4_MSG_AN)
 113                 rxq->stats.an++;
 114         else
 115                 rxq->stats.pkts++;
 116         return 0;
 117 }
 118
 119 static int alloc_uld_rxqs(struct adapter *adap,
 120                           struct sge_uld_rxq_info *rxq_info,
 121                           unsigned int nq, unsigned int offset, bool lro)
 122 {
 123         struct sge *s = &adap->sge;
 124         struct sge_ofld_rxq *q = rxq_info->uldrxq + offset;
 125         unsigned short *ids = rxq_info->rspq_id + offset;
 126         unsigned int per_chan = nq / adap->params.nports;
 127         unsigned int msi_idx, bmap_idx;
 128         int i, err;
 129
 130         if (adap->flags & USING_MSIX)
 131                 msi_idx = 1;
 132         else
 133                 msi_idx = -((int)s->intrq.abs_id + 1);
 134
 135         for (i = 0; i < nq; i++, q++) {
 136                 if (msi_idx >= 0) {
 137                         bmap_idx = get_msix_idx_from_bmap(adap);
 138                         adap->msi_idx++;
 139                 }
 140                 err = t4_sge_alloc_rxq(adap, &q->rspq, false,
 141                                        adap->port[i / per_chan],
 142                                        adap->msi_idx,
 143                                        q->fl.size ? &q->fl : NULL,
 144                                        uldrx_handler,
 145                                        NULL,
 146                                        0);
 147                 if (err)
 148                         goto freeout;
 149                 if (msi_idx >= 0)
 150                         rxq_info->msix_tbl[i + offset] = bmap_idx;
 151                 memset(&q->stats, 0, sizeof(q->stats));
 152                 if (ids)
 153                         ids[i] = q->rspq.abs_id;
 154         }
 155         return 0;
 156 freeout:
 157         q = rxq_info->uldrxq + offset;
 158         for ( ; i; i--, q++) {
 159                 if (q->rspq.desc)
 160                         free_rspq_fl(adap, &q->rspq,
 161                                      q->fl.size ? &q->fl : NULL);
 162                 adap->msi_idx--;
 163         }
 164
 165         /* We need to free rxq also in case of ciq allocation failure */
 166         if (offset) {
 167                 q = rxq_info->uldrxq + offset;
 168                 for ( ; i; i--, q++) {
 169                         if (q->rspq.desc)
 170                                 free_rspq_fl(adap, &q->rspq,
 171                                              q->fl.size ? &q->fl : NULL);
 172                         adap->msi_idx--;
 173                 }
 174         }
 175         return err;
 176 }
 177
 178 int setup_sge_queues_uld(struct adapter *adap, unsigned int uld_type, bool lro)
 179 {
 180         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 181
 182         if (adap->flags & USING_MSIX) {
 183                 rxq_info->msix_tbl = kzalloc(rxq_info->nrxq + rxq_info->nciq,
 184                                              GFP_KERNEL);
 185                 if (!rxq_info->msix_tbl)
 186                         return -ENOMEM;
 187         }
 188
 189         return !(!alloc_uld_rxqs(adap, rxq_info, rxq_info->nrxq, 0, lro) &&
 190                  !alloc_uld_rxqs(adap, rxq_info, rxq_info->nciq,
 191                                  rxq_info->nrxq, lro));
 192 }
 193
 194 static void t4_free_uld_rxqs(struct adapter *adap, int n,
 195                              struct sge_ofld_rxq *q)
 196 {
 197         for ( ; n; n--, q++) {
 198                 if (q->rspq.desc)
 199                         free_rspq_fl(adap, &q->rspq,
 200                                      q->fl.size ? &q->fl : NULL);
 201                 adap->msi_idx--;
 202         }
 203 }
 204
 205 void free_sge_queues_uld(struct adapter *adap, unsigned int uld_type)
 206 {
 207         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 208
 209         if (rxq_info->nciq)
 210                 t4_free_uld_rxqs(adap, rxq_info->nciq,
 211                                  rxq_info->uldrxq + rxq_info->nrxq);
 212         t4_free_uld_rxqs(adap, rxq_info->nrxq, rxq_info->uldrxq);
 213         if (adap->flags & USING_MSIX)
 214                 kfree(rxq_info->msix_tbl);
 215 }
 216
 217 int cfg_queues_uld(struct adapter *adap, unsigned int uld_type,
 218                    const struct cxgb4_pci_uld_info *uld_info)
 219 {
 220         struct sge *s = &adap->sge;
 221         struct sge_uld_rxq_info *rxq_info;
 222         int i, nrxq;
 223
 224         rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);
 225         if (!rxq_info)
 226                 return -ENOMEM;
 227
 228         if (uld_info->nrxq > s->nqs_per_uld)
 229                 rxq_info->nrxq = s->nqs_per_uld;
 230         else
 231                 rxq_info->nrxq = uld_info->nrxq;
 232         if (!uld_info->nciq)
 233                 rxq_info->nciq = 0;
 234         else if (uld_info->nciq && uld_info->nciq > s->nqs_per_uld)
 235                 rxq_info->nciq = s->nqs_per_uld;
 236         else
 237                 rxq_info->nciq = uld_info->nciq;
 238
 239         nrxq = rxq_info->nrxq + rxq_info->nciq; /* total rxq's */
 240         rxq_info->uldrxq = kcalloc(nrxq, sizeof(struct sge_ofld_rxq),
 241                                    GFP_KERNEL);
 242         if (!rxq_info->uldrxq) {
 243                 kfree(rxq_info);
 244                 return -ENOMEM;
 245         }
 246
 247         rxq_info->rspq_id = kcalloc(nrxq, sizeof(unsigned short), GFP_KERNEL);
 248         if (!rxq_info->uldrxq) {
 249                 kfree(rxq_info->uldrxq);
 250                 kfree(rxq_info);
 251                 return -ENOMEM;
 252         }
 253
 254         for (i = 0; i < rxq_info->nrxq; i++) {
 255                 struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
 256
 257                 init_rspq(adap, &r->rspq, 5, 1, uld_info->rxq_size, 64);
 258                 r->rspq.uld = uld_type;
 259                 r->fl.size = 72;
 260         }
 261
 262         for (i = rxq_info->nrxq; i < nrxq; i++) {
 263                 struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
 264
 265                 init_rspq(adap, &r->rspq, 5, 1, uld_info->ciq_size, 64);
 266                 r->rspq.uld = uld_type;
 267                 r->fl.size = 72;
 268         }
 269
 270         memcpy(rxq_info->name, uld_info->name, IFNAMSIZ);
 271         adap->sge.uld_rxq_info[uld_type] = rxq_info;
 272
 273         return 0;
 274 }
 275
 276 void free_queues_uld(struct adapter *adap, unsigned int uld_type)
 277 {
 278         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 279
 280         kfree(rxq_info->rspq_id);
 281         kfree(rxq_info->uldrxq);
 282         kfree(rxq_info);
 283 }
 284
 285 int request_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
 286 {
 287         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 288         int idx, bmap_idx, err = 0;
 289
 290         for_each_uldrxq(rxq_info, idx) {
 291                 bmap_idx = rxq_info->msix_tbl[idx];
 292                 err = request_irq(adap->msix_info_ulds[bmap_idx].vec,
 293                                   t4_sge_intr_msix, 0,
 294                                   adap->msix_info_ulds[bmap_idx].desc,
 295                                   &rxq_info->uldrxq[idx].rspq);
 296                 if (err)
 297                         goto unwind;
 298         }
 299         return 0;
 300 unwind:
 301         while (--idx >= 0) {
 302                 bmap_idx = rxq_info->msix_tbl[idx];
 303                 free_msix_idx_in_bmap(adap, bmap_idx);
 304                 free_irq(adap->msix_info_ulds[bmap_idx].vec,
 305                          &rxq_info->uldrxq[idx].rspq);
 306         }
 307         return err;
 308 }
 309
 310 void free_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
 311 {
 312         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 313         int idx;
 314
 315         for_each_uldrxq(rxq_info, idx) {
 316                 unsigned int bmap_idx = rxq_info->msix_tbl[idx];
 317
 318                 free_msix_idx_in_bmap(adap, bmap_idx);
 319                 free_irq(adap->msix_info_ulds[bmap_idx].vec,
 320                          &rxq_info->uldrxq[idx].rspq);
 321         }
 322 }
 323
 324 void name_msix_vecs_uld(struct adapter *adap, unsigned int uld_type)
 325 {
 326         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 327         int n = sizeof(adap->msix_info_ulds[0].desc);
 328         int idx;
 329
 330         for_each_uldrxq(rxq_info, idx) {
 331                 unsigned int bmap_idx = rxq_info->msix_tbl[idx];
 332
 333                 snprintf(adap->msix_info_ulds[bmap_idx].desc, n, "%s-%s%d",
 334                          adap->port[0]->name, rxq_info->name, idx);
 335         }
 336 }
 337
 338 static void enable_rx(struct adapter *adap, struct sge_rspq *q)
 339 {
 340         if (!q)
 341                 return;
 342
 343         if (q->handler) {
 344                 cxgb_busy_poll_init_lock(q);
 345                 napi_enable(&q->napi);
 346         }
 347         /* 0-increment GTS to start the timer and enable interrupts */
 348         t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
 349                      SEINTARM_V(q->intr_params) |
 350                      INGRESSQID_V(q->cntxt_id));
 351 }
 352
 353 static void quiesce_rx(struct adapter *adap, struct sge_rspq *q)
 354 {
 355         if (q && q->handler) {
 356                 napi_disable(&q->napi);
 357                 local_bh_disable();
 358                 while (!cxgb_poll_lock_napi(q))
 359                         mdelay(1);
 360                 local_bh_enable();
 361         }
 362 }
 363
 364 void enable_rx_uld(struct adapter *adap, unsigned int uld_type)
 365 {
 366         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 367         int idx;
 368
 369         for_each_uldrxq(rxq_info, idx)
 370                 enable_rx(adap, &rxq_info->uldrxq[idx].rspq);
 371 }
 372
 373 void quiesce_rx_uld(struct adapter *adap, unsigned int uld_type)
 374 {
 375         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 376         int idx;
 377
 378         for_each_uldrxq(rxq_info, idx)
 379                 quiesce_rx(adap, &rxq_info->uldrxq[idx].rspq);
 380 }
 381
 382 static void uld_queue_init(struct adapter *adap, unsigned int uld_type,
 383                            struct cxgb4_lld_info *lli)
 384 {
 385         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 386
 387         lli->rxq_ids = rxq_info->rspq_id;
 388         lli->nrxq = rxq_info->nrxq;
 389         lli->ciq_ids = rxq_info->rspq_id + rxq_info->nrxq;
 390         lli->nciq = rxq_info->nciq;
 391 }
 392
 393 int uld_mem_alloc(struct adapter *adap)
 394 {
 395         struct sge *s = &adap->sge;
 396
 397         adap->uld = kcalloc(adap->num_uld, sizeof(*adap->uld), GFP_KERNEL);
 398         if (!adap->uld)
 399                 return -ENOMEM;
 400
 401         s->uld_rxq_info = kzalloc(adap->num_uld *
 402                                   sizeof(struct sge_uld_rxq_info *),
 403                                   GFP_KERNEL);
 404         if (!s->uld_rxq_info)
 405                 goto err_uld;
 406
 407         return 0;
 408 err_uld:
 409         kfree(adap->uld);
 410         return -ENOMEM;
 411 }
 412
 413 void uld_mem_free(struct adapter *adap)
 414 {
 415         struct sge *s = &adap->sge;
 416
 417         kfree(s->uld_rxq_info);
 418         kfree(adap->uld);
 419 }
 420
 421 static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld)
 422 {
 423         int i;
 424
 425         lld->pdev = adap->pdev;
 426         lld->pf = adap->pf;
 427         lld->l2t = adap->l2t;
 428         lld->tids = &adap->tids;
 429         lld->ports = adap->port;
 430         lld->vr = &adap->vres;
 431         lld->mtus = adap->params.mtus;
 432         lld->ntxq = adap->sge.iscsiqsets;
 433         lld->nchan = adap->params.nports;
 434         lld->nports = adap->params.nports;
 435         lld->wr_cred = adap->params.ofldq_wr_cred;
 436         lld->adapter_type = adap->params.chip;
 437         lld->cclk_ps = 1000000000 / adap->params.vpd.cclk;
 438         lld->udb_density = 1 << adap->params.sge.eq_qpp;
 439         lld->ucq_density = 1 << adap->params.sge.iq_qpp;
 440         lld->filt_mode = adap->params.tp.vlan_pri_map;
 441         /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
 442         for (i = 0; i < NCHAN; i++)
 443                 lld->tx_modq[i] = i;
 444         lld->gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
 445         lld->db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
 446         lld->fw_vers = adap->params.fw_vers;
 447         lld->dbfifo_int_thresh = dbfifo_int_thresh;
 448         lld->sge_ingpadboundary = adap->sge.fl_align;
 449         lld->sge_egrstatuspagesize = adap->sge.stat_len;
 450         lld->sge_pktshift = adap->sge.pktshift;
 451         lld->enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
 452         lld->max_ordird_qp = adap->params.max_ordird_qp;
 453         lld->max_ird_adapter = adap->params.max_ird_adapter;
 454         lld->ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
 455         lld->nodeid = dev_to_node(adap->pdev_dev);
 456 }
 457
 458 static void uld_attach(struct adapter *adap, unsigned int uld)
 459 {
 460         void *handle;
 461         struct cxgb4_lld_info lli;
 462
 463         uld_init(adap, &lli);
 464         uld_queue_init(adap, uld, &lli);
 465
 466         handle = adap->uld[uld].add(&lli);
 467         if (IS_ERR(handle)) {
 468                 dev_warn(adap->pdev_dev,
 469                          "could not attach to the %s driver, error %ld\n",
 470                          adap->uld[uld].name, PTR_ERR(handle));
 471                 return;
 472         }
 473
 474         adap->uld[uld].handle = handle;
 475
 476         if (adap->flags & FULL_INIT_DONE)
 477                 adap->uld[uld].state_change(handle, CXGB4_STATE_UP);
 478 }
 479
 480 int cxgb4_register_pci_uld(enum cxgb4_pci_uld type,
 481                            struct cxgb4_pci_uld_info *p)
 482 {
 483         int ret = 0;
 484         struct adapter *adap;
 485
 486         if (type >= CXGB4_PCI_ULD_MAX)
 487                 return -EINVAL;
 488
 489         mutex_lock(&uld_mutex);
 490         list_for_each_entry(adap, &adapter_list, list_node) {
 491                 if (!is_pci_uld(adap))
 492                         continue;
 493                 ret = cfg_queues_uld(adap, type, p);
 494                 if (ret)
 495                         goto out;
 496                 ret = setup_sge_queues_uld(adap, type, p->lro);
 497                 if (ret)
 498                         goto free_queues;
 499                 if (adap->flags & USING_MSIX) {
 500                         name_msix_vecs_uld(adap, type);
 501                         ret = request_msix_queue_irqs_uld(adap, type);
 502                         if (ret)
 503                                 goto free_rxq;
 504                 }
 505                 if (adap->flags & FULL_INIT_DONE)
 506                         enable_rx_uld(adap, type);
 507                 if (adap->uld[type].add) {
 508                         ret = -EBUSY;
 509                         goto free_irq;
 510                 }
 511                 adap->uld[type] = *p;
 512                 uld_attach(adap, type);
 513         }
 514         mutex_unlock(&uld_mutex);
 515         return 0;
 516
 517 free_irq:
 518         if (adap->flags & USING_MSIX)
 519                 free_msix_queue_irqs_uld(adap, type);
 520 free_rxq:
 521         free_sge_queues_uld(adap, type);
 522 free_queues:
 523         free_queues_uld(adap, type);
 524 out:
 525         mutex_unlock(&uld_mutex);
 526         return ret;
 527 }
 528 EXPORT_SYMBOL(cxgb4_register_pci_uld);
 529
 530 int cxgb4_unregister_pci_uld(enum cxgb4_pci_uld type)
 531 {
 532         struct adapter *adap;
 533
 534         if (type >= CXGB4_PCI_ULD_MAX)
 535                 return -EINVAL;
 536
 537         mutex_lock(&uld_mutex);
 538         list_for_each_entry(adap, &adapter_list, list_node) {
 539                 if (!is_pci_uld(adap))
 540                         continue;
 541                 adap->uld[type].handle = NULL;
 542                 adap->uld[type].add = NULL;
 543                 if (adap->flags & FULL_INIT_DONE)
 544                         quiesce_rx_uld(adap, type);
 545                 if (adap->flags & USING_MSIX)
 546                         free_msix_queue_irqs_uld(adap, type);
 547                 free_sge_queues_uld(adap, type);
 548                 free_queues_uld(adap, type);
 549         }
 550         mutex_unlock(&uld_mutex);
 551
 552         return 0;
 553 }
 554 EXPORT_SYMBOL(cxgb4_unregister_pci_uld);