drivers/infiniband/hw/hns/hns_roce_hem.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/platform_device.h>
  35 #include "hns_roce_device.h"
  36 #include "hns_roce_hem.h"
  37 #include "hns_roce_common.h"
  38
  39 #define HW_SYNC_TIMEOUT_MSECS           500
  40 #define HW_SYNC_SLEEP_TIME_INTERVAL     20
  41
  42 #define HNS_ROCE_HEM_ALLOC_SIZE         (1 << 17)
  43 #define HNS_ROCE_TABLE_CHUNK_SIZE       (1 << 17)
  44
  45 #define DMA_ADDR_T_SHIFT                12
  46 #define BT_CMD_SYNC_SHIFT               31
  47 #define BT_BA_SHIFT                     32
  48
  49 struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages,
  50                                         gfp_t gfp_mask)
  51 {
  52         struct hns_roce_hem_chunk *chunk = NULL;
  53         struct hns_roce_hem *hem;
  54         struct scatterlist *mem;
  55         int order;
  56         void *buf;
  57
  58         WARN_ON(gfp_mask & __GFP_HIGHMEM);
  59
  60         hem = kmalloc(sizeof(*hem),
  61                       gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
  62         if (!hem)
  63                 return NULL;
  64
  65         hem->refcount = 0;
  66         INIT_LIST_HEAD(&hem->chunk_list);
  67
  68         order = get_order(HNS_ROCE_HEM_ALLOC_SIZE);
  69
  70         while (npages > 0) {
  71                 if (!chunk) {
  72                         chunk = kmalloc(sizeof(*chunk),
  73                                 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
  74                         if (!chunk)
  75                                 goto fail;
  76
  77                         sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
  78                         chunk->npages = 0;
  79                         chunk->nsg = 0;
  80                         list_add_tail(&chunk->list, &hem->chunk_list);
  81                 }
  82
  83                 while (1 << order > npages)
  84                         --order;
  85
  86                 /*
  87                 * Alloc memory one time. If failed, don't alloc small block
  88                 * memory, directly return fail.
  89                 */
  90                 mem = &chunk->mem[chunk->npages];
  91                 buf = dma_alloc_coherent(&hr_dev->pdev->dev, PAGE_SIZE << order,
  92                                 &sg_dma_address(mem), gfp_mask);
  93                 if (!buf)
  94                         goto fail;
  95
  96                 sg_set_buf(mem, buf, PAGE_SIZE << order);
  97                 WARN_ON(mem->offset);
  98                 sg_dma_len(mem) = PAGE_SIZE << order;
  99
 100                 ++chunk->npages;
 101                 ++chunk->nsg;
 102                 npages -= 1 << order;
 103         }
 104
 105         return hem;
 106
 107 fail:
 108         hns_roce_free_hem(hr_dev, hem);
 109         return NULL;
 110 }
 111
 112 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
 113 {
 114         struct hns_roce_hem_chunk *chunk, *tmp;
 115         int i;
 116
 117         if (!hem)
 118                 return;
 119
 120         list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
 121                 for (i = 0; i < chunk->npages; ++i)
 122                         dma_free_coherent(&hr_dev->pdev->dev,
 123                                    chunk->mem[i].length,
 124                                    lowmem_page_address(sg_page(&chunk->mem[i])),
 125                                    sg_dma_address(&chunk->mem[i]));
 126                 kfree(chunk);
 127         }
 128
 129         kfree(hem);
 130 }
 131
 132 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
 133                             struct hns_roce_hem_table *table, unsigned long obj)
 134 {
 135         struct device *dev = &hr_dev->pdev->dev;
 136         spinlock_t *lock = &hr_dev->bt_cmd_lock;
 137         unsigned long end = 0;
 138         unsigned long flags;
 139         struct hns_roce_hem_iter iter;
 140         void __iomem *bt_cmd;
 141         u32 bt_cmd_h_val = 0;
 142         u32 bt_cmd_val[2];
 143         u32 bt_cmd_l = 0;
 144         u64 bt_ba = 0;
 145         int ret = 0;
 146
 147         /* Find the HEM(Hardware Entry Memory) entry */
 148         unsigned long i = (obj & (table->num_obj - 1)) /
 149                           (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size);
 150
 151         switch (table->type) {
 152         case HEM_TYPE_QPC:
 153                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 154                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
 155                 break;
 156         case HEM_TYPE_MTPT:
 157                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 158                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 159                                HEM_TYPE_MTPT);
 160                 break;
 161         case HEM_TYPE_CQC:
 162                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 163                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
 164                 break;
 165         case HEM_TYPE_SRQC:
 166                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 167                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 168                                HEM_TYPE_SRQC);
 169                 break;
 170         default:
 171                 return ret;
 172         }
 173         roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
 174                        ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
 175         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
 176         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
 177
 178         /* Currently iter only a chunk */
 179         for (hns_roce_hem_first(table->hem[i], &iter);
 180              !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
 181                 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
 182
 183                 spin_lock_irqsave(lock, flags);
 184
 185                 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
 186
 187                 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
 188                 while (1) {
 189                         if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
 190                                 if (!(time_before(jiffies, end))) {
 191                                         dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
 192                                         spin_unlock_irqrestore(lock, flags);
 193                                         return -EBUSY;
 194                                 }
 195                         } else {
 196                                 break;
 197                         }
 198                         msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
 199                 }
 200
 201                 bt_cmd_l = (u32)bt_ba;
 202                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
 203                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
 204                                bt_ba >> BT_BA_SHIFT);
 205
 206                 bt_cmd_val[0] = bt_cmd_l;
 207                 bt_cmd_val[1] = bt_cmd_h_val;
 208                 hns_roce_write64_k(bt_cmd_val,
 209                                    hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
 210                 spin_unlock_irqrestore(lock, flags);
 211         }
 212
 213         return ret;
 214 }
 215
 216 static int hns_roce_clear_hem(struct hns_roce_dev *hr_dev,
 217                               struct hns_roce_hem_table *table,
 218                               unsigned long obj)
 219 {
 220         struct device *dev = &hr_dev->pdev->dev;
 221         unsigned long end = 0;
 222         unsigned long flags;
 223         void __iomem *bt_cmd;
 224         uint32_t bt_cmd_val[2];
 225         u32 bt_cmd_h_val = 0;
 226         int ret = 0;
 227
 228         switch (table->type) {
 229         case HEM_TYPE_QPC:
 230                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 231                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
 232                 break;
 233         case HEM_TYPE_MTPT:
 234                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 235                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 236                                HEM_TYPE_MTPT);
 237                 break;
 238         case HEM_TYPE_CQC:
 239                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 240                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
 241                 break;
 242         case HEM_TYPE_SRQC:
 243                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 244                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 245                                HEM_TYPE_SRQC);
 246                 break;
 247         default:
 248                 return ret;
 249         }
 250         roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
 251                        ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
 252         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
 253         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
 254         roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
 255                        ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, 0);
 256
 257         spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags);
 258
 259         bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
 260
 261         end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
 262         while (1) {
 263                 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
 264                         if (!(time_before(jiffies, end))) {
 265                                 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
 266                                 spin_unlock_irqrestore(&hr_dev->bt_cmd_lock,
 267                                                        flags);
 268                                 return -EBUSY;
 269                         }
 270                 } else {
 271                         break;
 272                 }
 273                 msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
 274         }
 275
 276         bt_cmd_val[0] = 0;
 277         bt_cmd_val[1] = bt_cmd_h_val;
 278         hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
 279         spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags);
 280
 281         return ret;
 282 }
 283
 284 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
 285                        struct hns_roce_hem_table *table, unsigned long obj)
 286 {
 287         struct device *dev = &hr_dev->pdev->dev;
 288         int ret = 0;
 289         unsigned long i;
 290
 291         i = (obj & (table->num_obj - 1)) / (HNS_ROCE_TABLE_CHUNK_SIZE /
 292              table->obj_size);
 293
 294         mutex_lock(&table->mutex);
 295
 296         if (table->hem[i]) {
 297                 ++table->hem[i]->refcount;
 298                 goto out;
 299         }
 300
 301         table->hem[i] = hns_roce_alloc_hem(hr_dev,
 302                                        HNS_ROCE_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
 303                                        (table->lowmem ? GFP_KERNEL :
 304                                         GFP_HIGHUSER) | __GFP_NOWARN);
 305         if (!table->hem[i]) {
 306                 ret = -ENOMEM;
 307                 goto out;
 308         }
 309
 310         /* Set HEM base address(128K/page, pa) to Hardware */
 311         if (hns_roce_set_hem(hr_dev, table, obj)) {
 312                 ret = -ENODEV;
 313                 dev_err(dev, "set HEM base address to HW failed.\n");
 314                 goto out;
 315         }
 316
 317         ++table->hem[i]->refcount;
 318 out:
 319         mutex_unlock(&table->mutex);
 320         return ret;
 321 }
 322
 323 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
 324                         struct hns_roce_hem_table *table, unsigned long obj)
 325 {
 326         struct device *dev = &hr_dev->pdev->dev;
 327         unsigned long i;
 328
 329         i = (obj & (table->num_obj - 1)) /
 330             (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size);
 331
 332         mutex_lock(&table->mutex);
 333
 334         if (--table->hem[i]->refcount == 0) {
 335                 /* Clear HEM base address */
 336                 if (hns_roce_clear_hem(hr_dev, table, obj))
 337                         dev_warn(dev, "Clear HEM base address failed.\n");
 338
 339                 hns_roce_free_hem(hr_dev, table->hem[i]);
 340                 table->hem[i] = NULL;
 341         }
 342
 343         mutex_unlock(&table->mutex);
 344 }
 345
 346 void *hns_roce_table_find(struct hns_roce_hem_table *table, unsigned long obj,
 347                           dma_addr_t *dma_handle)
 348 {
 349         struct hns_roce_hem_chunk *chunk;
 350         unsigned long idx;
 351         int i;
 352         int offset, dma_offset;
 353         struct hns_roce_hem *hem;
 354         struct page *page = NULL;
 355
 356         if (!table->lowmem)
 357                 return NULL;
 358
 359         mutex_lock(&table->mutex);
 360         idx = (obj & (table->num_obj - 1)) * table->obj_size;
 361         hem = table->hem[idx / HNS_ROCE_TABLE_CHUNK_SIZE];
 362         dma_offset = offset = idx % HNS_ROCE_TABLE_CHUNK_SIZE;
 363
 364         if (!hem)
 365                 goto out;
 366
 367         list_for_each_entry(chunk, &hem->chunk_list, list) {
 368                 for (i = 0; i < chunk->npages; ++i) {
 369                         if (dma_handle && dma_offset >= 0) {
 370                                 if (sg_dma_len(&chunk->mem[i]) >
 371                                     (u32)dma_offset)
 372                                         *dma_handle = sg_dma_address(
 373                                                 &chunk->mem[i]) + dma_offset;
 374                                 dma_offset -= sg_dma_len(&chunk->mem[i]);
 375                         }
 376
 377                         if (chunk->mem[i].length > (u32)offset) {
 378                                 page = sg_page(&chunk->mem[i]);
 379                                 goto out;
 380                         }
 381                         offset -= chunk->mem[i].length;
 382                 }
 383         }
 384
 385 out:
 386         mutex_unlock(&table->mutex);
 387         return page ? lowmem_page_address(page) + offset : NULL;
 388 }
 389
 390 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
 391                              struct hns_roce_hem_table *table,
 392                              unsigned long start, unsigned long end)
 393 {
 394         unsigned long inc = HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size;
 395         unsigned long i = 0;
 396         int ret = 0;
 397
 398         /* Allocate MTT entry memory according to chunk(128K) */
 399         for (i = start; i <= end; i += inc) {
 400                 ret = hns_roce_table_get(hr_dev, table, i);
 401                 if (ret)
 402                         goto fail;
 403         }
 404
 405         return 0;
 406
 407 fail:
 408         while (i > start) {
 409                 i -= inc;
 410                 hns_roce_table_put(hr_dev, table, i);
 411         }
 412         return ret;
 413 }
 414
 415 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
 416                               struct hns_roce_hem_table *table,
 417                               unsigned long start, unsigned long end)
 418 {
 419         unsigned long i;
 420
 421         for (i = start; i <= end;
 422                 i += HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size)
 423                 hns_roce_table_put(hr_dev, table, i);
 424 }
 425
 426 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 427                             struct hns_roce_hem_table *table, u32 type,
 428                             unsigned long obj_size, unsigned long nobj,
 429                             int use_lowmem)
 430 {
 431         unsigned long obj_per_chunk;
 432         unsigned long num_hem;
 433
 434         obj_per_chunk = HNS_ROCE_TABLE_CHUNK_SIZE / obj_size;
 435         num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
 436
 437         table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
 438         if (!table->hem)
 439                 return -ENOMEM;
 440
 441         table->type = type;
 442         table->num_hem = num_hem;
 443         table->num_obj = nobj;
 444         table->obj_size = obj_size;
 445         table->lowmem = use_lowmem;
 446         mutex_init(&table->mutex);
 447
 448         return 0;
 449 }
 450
 451 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
 452                                 struct hns_roce_hem_table *table)
 453 {
 454         struct device *dev = &hr_dev->pdev->dev;
 455         unsigned long i;
 456
 457         for (i = 0; i < table->num_hem; ++i)
 458                 if (table->hem[i]) {
 459                         if (hns_roce_clear_hem(hr_dev, table,
 460                             i * HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size))
 461                                 dev_err(dev, "Clear HEM base address failed.\n");
 462
 463                         hns_roce_free_hem(hr_dev, table->hem[i]);
 464                 }
 465
 466         kfree(table->hem);
 467 }
 468
 469 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
 470 {
 471         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
 472         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
 473         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
 474         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
 475         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
 476 }