projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/cma: Accept connection without a valid netdev on RoCE
[deliverable/linux.git] / drivers / infiniband / hw / nes / nes_verbs.c
CommitLineData
3c2d774c 1/*
c5488c57 2 * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
3c2d774c
GS		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34#include <linux/module.h>
35#include <linux/moduleparam.h>
36#include <linux/random.h>
37#include <linux/highmem.h>
5a0e3ad6 38#include <linux/slab.h>
3c2d774c
GS		 39#include <asm/byteorder.h>
40
41#include <rdma/ib_verbs.h>
42#include <rdma/iw_cm.h>
43#include <rdma/ib_user_verbs.h>
44
45#include "nes.h"
46
47#include <rdma/ib_umem.h>
48
49atomic_t mod_qp_timouts;
50atomic_t qps_created;
51atomic_t sw_qps_destroyed;
52
1a855fbf 53static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev);
3c2d774c
GS		 54
55/**
56 * nes_alloc_mw
57 */
7083e42e
SM		 58static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd, enum ib_mw_type type)
59{
3c2d774c
GS		 60 struct nes_pd *nespd = to_nespd(ibpd);
61 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
62 struct nes_device *nesdev = nesvnic->nesdev;
63 struct nes_adapter *nesadapter = nesdev->nesadapter;
64 struct nes_cqp_request *cqp_request;
65 struct nes_mr *nesmr;
66 struct ib_mw *ibmw;
67 struct nes_hw_cqp_wqe *cqp_wqe;
68 int ret;
69 u32 stag;
70 u32 stag_index = 0;
71 u32 next_stag_index = 0;
72 u32 driver_key = 0;
73 u8 stag_key = 0;
74
7083e42e
SM		 75 if (type != IB_MW_TYPE_1)
76 return ERR_PTR(-EINVAL);
77
3c2d774c
GS		 78 get_random_bytes(&next_stag_index, sizeof(next_stag_index));
79 stag_key = (u8)next_stag_index;
80
81 driver_key = 0;
82
83 next_stag_index >>= 8;
84 next_stag_index %= nesadapter->max_mr;
85
86 ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
81821644 87 nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_MW);
3c2d774c
GS		 88 if (ret) {
89 return ERR_PTR(ret);
90 }
91
92 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
93 if (!nesmr) {
94 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
95 return ERR_PTR(-ENOMEM);
96 }
97
98 stag = stag_index << 8;
99 stag |= driver_key;
100 stag += (u32)stag_key;
101
102 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n",
103 stag, stag_index);
104
105 /* Register the region with the adapter */
106 cqp_request = nes_get_cqp_request(nesdev);
107 if (cqp_request == NULL) {
108 kfree(nesmr);
109 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
110 return ERR_PTR(-ENOMEM);
111 }
112
113 cqp_request->waiting = 1;
114 cqp_wqe = &cqp_request->cqp_wqe;
115
116 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
117 cpu_to_le32( NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_RIGHTS_REMOTE_READ |
118 NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_VA_TO |
119 NES_CQP_STAG_REM_ACC_EN);
120
121 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
122 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff));
123 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
124
125 atomic_set(&cqp_request->refcount, 2);
8294f297 126 nes_post_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 127
128 /* Wait for CQP */
129 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
130 NES_EVENT_TIMEOUT);
131 nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
132 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
133 stag, ret, cqp_request->major_code, cqp_request->minor_code);
134 if ((!ret) || (cqp_request->major_code)) {
1ff66e8c 135 nes_put_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 136 kfree(nesmr);
137 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
138 if (!ret) {
139 return ERR_PTR(-ETIME);
140 } else {
141 return ERR_PTR(-ENOMEM);
142 }
3c2d774c 143 }
1ff66e8c 144 nes_put_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 145
146 nesmr->ibmw.rkey = stag;
147 nesmr->mode = IWNES_MEMREG_TYPE_MW;
148 ibmw = &nesmr->ibmw;
149 nesmr->pbl_4k = 0;
150 nesmr->pbls_used = 0;
151
152 return ibmw;
153}
154
155
156/**
157 * nes_dealloc_mw
158 */
159static int nes_dealloc_mw(struct ib_mw *ibmw)
160{
161 struct nes_mr *nesmr = to_nesmw(ibmw);
162 struct nes_vnic *nesvnic = to_nesvnic(ibmw->device);
163 struct nes_device *nesdev = nesvnic->nesdev;
164 struct nes_adapter *nesadapter = nesdev->nesadapter;
165 struct nes_hw_cqp_wqe *cqp_wqe;
166 struct nes_cqp_request *cqp_request;
167 int err = 0;
3c2d774c
GS		 168 int ret;
169
170 /* Deallocate the window with the adapter */
171 cqp_request = nes_get_cqp_request(nesdev);
172 if (cqp_request == NULL) {
173 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
174 return -ENOMEM;
175 }
176 cqp_request->waiting = 1;
177 cqp_wqe = &cqp_request->cqp_wqe;
178 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
179 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, NES_CQP_DEALLOCATE_STAG);
180 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ibmw->rkey);
181
182 atomic_set(&cqp_request->refcount, 2);
8294f297 183 nes_post_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 184
185 /* Wait for CQP */
186 nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X to complete.\n",
187 ibmw->rkey);
188 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done),
189 NES_EVENT_TIMEOUT);
190 nes_debug(NES_DBG_MR, "Deallocate STag completed, wait_event_timeout ret = %u,"
191 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
192 ret, cqp_request->major_code, cqp_request->minor_code);
1ff66e8c
RD		 193 if (!ret)
194 err = -ETIME;
195 else if (cqp_request->major_code)
196 err = -EIO;
197
198 nes_put_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 199
200 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
201 (ibmw->rkey & 0x0fffff00) >> 8);
202 kfree(nesmr);
203
204 return err;
205}
206
207
208/**
209 * nes_bind_mw
210 */
211static int nes_bind_mw(struct ib_qp *ibqp, struct ib_mw *ibmw,
212 struct ib_mw_bind *ibmw_bind)
213{
214 u64 u64temp;
215 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device);
216 struct nes_device *nesdev = nesvnic->nesdev;
217 /* struct nes_mr *nesmr = to_nesmw(ibmw); */
218 struct nes_qp *nesqp = to_nesqp(ibqp);
219 struct nes_hw_qp_wqe *wqe;
220 unsigned long flags = 0;
221 u32 head;
222 u32 wqe_misc = 0;
223 u32 qsize;
224
225 if (nesqp->ibqp_state > IB_QPS_RTS)
226 return -EINVAL;
227
5880ff19 228 spin_lock_irqsave(&nesqp->lock, flags);
3c2d774c
GS		 229
230 head = nesqp->hwqp.sq_head;
231 qsize = nesqp->hwqp.sq_tail;
232
233 /* Check for SQ overflow */
234 if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) {
5880ff19 235 spin_unlock_irqrestore(&nesqp->lock, flags);
831d06cf 236 return -ENOMEM;
3c2d774c
GS		 237 }
238
239 wqe = &nesqp->hwqp.sq_vbase[head];
240 /* nes_debug(NES_DBG_MR, "processing sq wqe at %p, head = %u.\n", wqe, head); */
241 nes_fill_init_qp_wqe(wqe, nesqp, head);
242 u64temp = ibmw_bind->wr_id;
243 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, u64temp);
244 wqe_misc = NES_IWARP_SQ_OP_BIND;
245
246 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
247
248 if (ibmw_bind->send_flags & IB_SEND_SIGNALED)
249 wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL;
250
7083e42e 251 if (ibmw_bind->bind_info.mw_access_flags & IB_ACCESS_REMOTE_WRITE)
3c2d774c 252 wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE;
7083e42e 253 if (ibmw_bind->bind_info.mw_access_flags & IB_ACCESS_REMOTE_READ)
3c2d774c 254 wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_READ;
3c2d774c
GS		 255
256 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_MISC_IDX, wqe_misc);
7083e42e
SM		 257 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MR_IDX,
258 ibmw_bind->bind_info.mr->lkey);
3c2d774c
GS		 259 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MW_IDX, ibmw->rkey);
260 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_LENGTH_LOW_IDX,
7083e42e 261 ibmw_bind->bind_info.length);
3c2d774c 262 wqe->wqe_words[NES_IWARP_SQ_BIND_WQE_LENGTH_HIGH_IDX] = 0;
7083e42e 263 u64temp = (u64)ibmw_bind->bind_info.addr;
3c2d774c
GS		 264 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_VA_FBO_LOW_IDX, u64temp);
265
266 head++;
267 if (head >= qsize)
268 head = 0;
269
270 nesqp->hwqp.sq_head = head;
271 barrier();
272
273 nes_write32(nesdev->regs+NES_WQE_ALLOC,
274 (1 << 24) | 0x00800000 | nesqp->hwqp.qp_id);
275
5880ff19 276 spin_unlock_irqrestore(&nesqp->lock, flags);
3c2d774c
GS		 277
278 return 0;
279}
280
281
e293a26f
CT		 282/*
283 * nes_alloc_fast_mr
3c2d774c 284 */
e293a26f
CT		 285static int alloc_fast_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
286 u32 stag, u32 page_count)
3c2d774c 287{
3c2d774c 288 struct nes_hw_cqp_wqe *cqp_wqe;
e293a26f
CT		 289 struct nes_cqp_request *cqp_request;
290 unsigned long flags;
3c2d774c 291 int ret;
e293a26f 292 struct nes_adapter *nesadapter = nesdev->nesadapter;
3c2d774c 293 u32 opcode = 0;
e293a26f
CT		 294 u16 major_code;
295 u64 region_length = page_count * PAGE_SIZE;
3c2d774c 296
3c2d774c 297
3c2d774c
GS		 298 cqp_request = nes_get_cqp_request(nesdev);
299 if (cqp_request == NULL) {
300 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
e293a26f 301 return -ENOMEM;
3c2d774c 302 }
e293a26f
CT		 303 nes_debug(NES_DBG_MR, "alloc_fast_reg_mr: page_count = %d, "
304 "region_length = %llu\n",
305 page_count, region_length);
3c2d774c
GS		 306 cqp_request->waiting = 1;
307 cqp_wqe = &cqp_request->cqp_wqe;
308
e293a26f
CT		 309 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
310 if (nesadapter->free_4kpbl > 0) {
311 nesadapter->free_4kpbl--;
312 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
313 } else {
314 /* No 4kpbl's available: */
315 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
316 nes_debug(NES_DBG_MR, "Out of Pbls\n");
317 nes_free_cqp_request(nesdev, cqp_request);
318 return -ENOMEM;
3c2d774c
GS		 319 }
320
e293a26f
CT		 321 opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_MR |
322 NES_CQP_STAG_PBL_BLK_SIZE | NES_CQP_STAG_VA_TO |
323 NES_CQP_STAG_REM_ACC_EN;
324 /*
325 * The current OFED API does not support the zero based TO option.
326 * If added then need to changed the NES_CQP_STAG_VA* option. Also,
327 * the API does not support that ability to have the MR set for local
328 * access only when created and not allow the SQ op to override. Given
329 * this the remote enable must be set here.
330 */
3c2d774c
GS		 331
332 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
333 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
e293a26f 334 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, 1);
3c2d774c 335
e293a26f
CT		 336 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] =
337 cpu_to_le32((u32)(region_length >> 8) & 0xff000000);
338 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |=
339 cpu_to_le32(nespd->pd_id & 0x00007fff);
340
341 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
342 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, 0);
343 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, 0);
344 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, 0);
345 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (page_count * 8));
346 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
347 barrier();
3c2d774c
GS		 348
349 atomic_set(&cqp_request->refcount, 2);
8294f297 350 nes_post_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 351
352 /* Wait for CQP */
e293a26f
CT		 353 ret = wait_event_timeout(cqp_request->waitq,
354 (0 != cqp_request->request_done),
355 NES_EVENT_TIMEOUT);
356
357 nes_debug(NES_DBG_MR, "Allocate STag 0x%08X completed, "
358 "wait_event_timeout ret = %u, CQP Major:Minor codes = "
359 "0x%04X:0x%04X.\n", stag, ret, cqp_request->major_code,
360 cqp_request->minor_code);
361 major_code = cqp_request->major_code;
1ff66e8c 362 nes_put_cqp_request(nesdev, cqp_request);
3c2d774c 363
e293a26f 364 if (!ret || major_code) {
35c6d694 365 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
e293a26f 366 nesadapter->free_4kpbl++;
35c6d694
CT		 367 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
368 }
369
e293a26f
CT		 370 if (!ret)
371 return -ETIME;
372 else if (major_code)
373 return -EIO;
374 return 0;
3c2d774c
GS		 375}
376
e293a26f 377/*
e02e4d55 378 * nes_alloc_mr
3c2d774c 379 */
e02e4d55
SG		 380static struct ib_mr *nes_alloc_mr(struct ib_pd *ibpd,
381 enum ib_mr_type mr_type,
382 u32 max_num_sg)
3c2d774c 383{
e293a26f
CT		 384 struct nes_pd *nespd = to_nespd(ibpd);
385 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
3c2d774c 386 struct nes_device *nesdev = nesvnic->nesdev;
35c6d694 387 struct nes_adapter *nesadapter = nesdev->nesadapter;
3c2d774c 388
e293a26f
CT		 389 u32 next_stag_index;
390 u8 stag_key = 0;
391 u32 driver_key = 0;
392 int err = 0;
393 u32 stag_index = 0;
394 struct nes_mr *nesmr;
395 u32 stag;
396 int ret;
397 struct ib_mr *ibmr;
e02e4d55
SG		 398
399 if (mr_type != IB_MR_TYPE_MEM_REG)
400 return ERR_PTR(-EINVAL);
401
402 if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
403 return ERR_PTR(-E2BIG);
404
e293a26f
CT		 405/*
406 * Note: Set to always use a fixed length single page entry PBL. This is to allow
407 * for the fast_reg_mr operation to always know the size of the PBL.
408 */
e02e4d55 409 if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
e293a26f 410 return ERR_PTR(-E2BIG);
35c6d694 411
e293a26f
CT		 412 get_random_bytes(&next_stag_index, sizeof(next_stag_index));
413 stag_key = (u8)next_stag_index;
414 next_stag_index >>= 8;
415 next_stag_index %= nesadapter->max_mr;
dae5d13a 416
e293a26f
CT		 417 err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
418 nesadapter->max_mr, &stag_index,
81821644 419 &next_stag_index, NES_RESOURCE_FAST_MR);
e293a26f
CT		 420 if (err)
421 return ERR_PTR(err);
422
423 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
424 if (!nesmr) {
425 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
426 return ERR_PTR(-ENOMEM);
35c6d694 427 }
3c2d774c 428
e293a26f
CT		 429 stag = stag_index << 8;
430 stag |= driver_key;
431 stag += (u32)stag_key;
3c2d774c 432
e293a26f
CT		 433 nes_debug(NES_DBG_MR, "Allocating STag 0x%08X index = 0x%08X\n",
434 stag, stag_index);
3c2d774c 435
e02e4d55 436 ret = alloc_fast_reg_mr(nesdev, nespd, stag, max_num_sg);
e293a26f
CT		 437
438 if (ret == 0) {
439 nesmr->ibmr.rkey = stag;
440 nesmr->ibmr.lkey = stag;
441 nesmr->mode = IWNES_MEMREG_TYPE_FMEM;
442 ibmr = &nesmr->ibmr;
443 } else {
444 kfree(nesmr);
445 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
446 ibmr = ERR_PTR(-ENOMEM);
447 }
448 return ibmr;
449}
450
451/*
452 * nes_alloc_fast_reg_page_list
3c2d774c 453 */
e293a26f
CT		 454static struct ib_fast_reg_page_list *nes_alloc_fast_reg_page_list(
455 struct ib_device *ibdev,
456 int page_list_len)
3c2d774c 457{
e293a26f
CT		 458 struct nes_vnic *nesvnic = to_nesvnic(ibdev);
459 struct nes_device *nesdev = nesvnic->nesdev;
460 struct ib_fast_reg_page_list *pifrpl;
461 struct nes_ib_fast_reg_page_list *pnesfrpl;
3c2d774c 462
e293a26f
CT		 463 if (page_list_len > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
464 return ERR_PTR(-E2BIG);
465 /*
466 * Allocate the ib_fast_reg_page_list structure, the
467 * nes_fast_bpl structure, and the PLB table.
468 */
469 pnesfrpl = kmalloc(sizeof(struct nes_ib_fast_reg_page_list) +
470 page_list_len * sizeof(u64), GFP_KERNEL);
471
472 if (!pnesfrpl)
473 return ERR_PTR(-ENOMEM);
3c2d774c 474
e293a26f
CT		 475 pifrpl = &pnesfrpl->ibfrpl;
476 pifrpl->page_list = &pnesfrpl->pbl;
477 pifrpl->max_page_list_len = page_list_len;
478 /*
479 * Allocate the WQE PBL
480 */
481 pnesfrpl->nes_wqe_pbl.kva = pci_alloc_consistent(nesdev->pcidev,
482 page_list_len * sizeof(u64),
483 &pnesfrpl->nes_wqe_pbl.paddr);
484
485 if (!pnesfrpl->nes_wqe_pbl.kva) {
486 kfree(pnesfrpl);
487 return ERR_PTR(-ENOMEM);
488 }
489 nes_debug(NES_DBG_MR, "nes_alloc_fast_reg_pbl: nes_frpl = %p, "
490 "ibfrpl = %p, ibfrpl.page_list = %p, pbl.kva = %p, "
183ae74b 491 "pbl.paddr = %llx\n", pnesfrpl, &pnesfrpl->ibfrpl,
e293a26f 492 pnesfrpl->ibfrpl.page_list, pnesfrpl->nes_wqe_pbl.kva,
183ae74b 493 (unsigned long long) pnesfrpl->nes_wqe_pbl.paddr);
e293a26f
CT		 494
495 return pifrpl;
496}
497
498/*
499 * nes_free_fast_reg_page_list
3c2d774c 500 */
e293a26f 501static void nes_free_fast_reg_page_list(struct ib_fast_reg_page_list *pifrpl)
3c2d774c 502{
e293a26f
CT		 503 struct nes_vnic *nesvnic = to_nesvnic(pifrpl->device);
504 struct nes_device *nesdev = nesvnic->nesdev;
505 struct nes_ib_fast_reg_page_list *pnesfrpl;
506
507 pnesfrpl = container_of(pifrpl, struct nes_ib_fast_reg_page_list, ibfrpl);
508 /*
509 * Free the WQE PBL.
510 */
511 pci_free_consistent(nesdev->pcidev,
512 pifrpl->max_page_list_len * sizeof(u64),
513 pnesfrpl->nes_wqe_pbl.kva,
514 pnesfrpl->nes_wqe_pbl.paddr);
515 /*
516 * Free the PBL structure
517 */
518 kfree(pnesfrpl);
3c2d774c
GS		 519}
520
3c2d774c
GS		 521/**
522 * nes_query_device
523 */
2528e33e
MB		 524static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
525 struct ib_udata *uhw)
3c2d774c
GS		 526{
527 struct nes_vnic *nesvnic = to_nesvnic(ibdev);
528 struct nes_device *nesdev = nesvnic->nesdev;
529 struct nes_ib_device *nesibdev = nesvnic->nesibdev;
530
2528e33e
MB		 531 if (uhw->inlen || uhw->outlen)
532 return -EINVAL;
533
3c2d774c
GS		 534 memset(props, 0, sizeof(*props));
535 memcpy(&props->sys_image_guid, nesvnic->netdev->dev_addr, 6);
536
ff0380ce 537 props->fw_ver = nesdev->nesadapter->firmware_version;
3c2d774c
GS		 538 props->device_cap_flags = nesdev->nesadapter->device_cap_flags;
539 props->vendor_id = nesdev->nesadapter->vendor_id;
540 props->vendor_part_id = nesdev->nesadapter->vendor_part_id;
541 props->hw_ver = nesdev->nesadapter->hw_rev;
542 props->max_mr_size = 0x80000000;
543 props->max_qp = nesibdev->max_qp;
544 props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2;
545 props->max_sge = nesdev->nesadapter->max_sge;
546 props->max_cq = nesibdev->max_cq;
5924aea6 547 props->max_cqe = nesdev->nesadapter->max_cqe;
3c2d774c
GS		 548 props->max_mr = nesibdev->max_mr;
549 props->max_mw = nesibdev->max_mr;
550 props->max_pd = nesibdev->max_pd;
551 props->max_sge_rd = 1;
552 switch (nesdev->nesadapter->max_irrq_wr) {
553 case 0:
8ac7f6e1 554 props->max_qp_rd_atom = 2;
3c2d774c
GS		 555 break;
556 case 1:
8ac7f6e1 557 props->max_qp_rd_atom = 8;
3c2d774c
GS		 558 break;
559 case 2:
8ac7f6e1 560 props->max_qp_rd_atom = 32;
3c2d774c
GS		 561 break;
562 case 3:
8ac7f6e1 563 props->max_qp_rd_atom = 64;
3c2d774c
GS		 564 break;
565 default:
566 props->max_qp_rd_atom = 0;
567 }
66388d67 568 props->max_qp_init_rd_atom = props->max_qp_rd_atom;
3c2d774c
GS		 569 props->atomic_cap = IB_ATOMIC_NONE;
570 props->max_map_per_fmr = 1;
571
572 return 0;
573}
574
575
576/**
577 * nes_query_port
578 */
579static int nes_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props)
580{
cd1d3f7a
CT		 581 struct nes_vnic *nesvnic = to_nesvnic(ibdev);
582 struct net_device *netdev = nesvnic->netdev;
583
3c2d774c
GS		 584 memset(props, 0, sizeof(*props));
585
cd1d3f7a
CT		 586 props->max_mtu = IB_MTU_4096;
587
588 if (netdev->mtu >= 4096)
589 props->active_mtu = IB_MTU_4096;
590 else if (netdev->mtu >= 2048)
591 props->active_mtu = IB_MTU_2048;
592 else if (netdev->mtu >= 1024)
593 props->active_mtu = IB_MTU_1024;
594 else if (netdev->mtu >= 512)
595 props->active_mtu = IB_MTU_512;
596 else
597 props->active_mtu = IB_MTU_256;
598
3c2d774c
GS		 599 props->lid = 1;
600 props->lmc = 0;
601 props->sm_lid = 0;
602 props->sm_sl = 0;
29327721
CT		 603 if (netif_queue_stopped(netdev))
604 props->state = IB_PORT_DOWN;
605 else if (nesvnic->linkup)
cd1d3f7a
CT		 606 props->state = IB_PORT_ACTIVE;
607 else
608 props->state = IB_PORT_DOWN;
3c2d774c
GS		 609 props->phys_state = 0;
610 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
611 IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
612 props->gid_tbl_len = 1;
613 props->pkey_tbl_len = 1;
614 props->qkey_viol_cntr = 0;
615 props->active_width = IB_WIDTH_4X;
2e96691c 616 props->active_speed = IB_SPEED_SDR;
3c2d774c
GS		 617 props->max_msg_sz = 0x80000000;
618
619 return 0;
620}
621
3c2d774c
GS		 622/**
623 * nes_query_pkey
624 */
625static int nes_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
626{
627 *pkey = 0;
628 return 0;
629}
630
631
632/**
633 * nes_query_gid
634 */
635static int nes_query_gid(struct ib_device *ibdev, u8 port,
636 int index, union ib_gid *gid)
637{
638 struct nes_vnic *nesvnic = to_nesvnic(ibdev);
639
640 memset(&(gid->raw[0]), 0, sizeof(gid->raw));
641 memcpy(&(gid->raw[0]), nesvnic->netdev->dev_addr, 6);
642
643 return 0;
644}
645
646
647/**
648 * nes_alloc_ucontext - Allocate the user context data structure. This keeps track
649 * of all objects associated with a particular user-mode client.
650 */
651static struct ib_ucontext *nes_alloc_ucontext(struct ib_device *ibdev,
652 struct ib_udata *udata)
653{
654 struct nes_vnic *nesvnic = to_nesvnic(ibdev);
655 struct nes_device *nesdev = nesvnic->nesdev;
656 struct nes_adapter *nesadapter = nesdev->nesadapter;
657 struct nes_alloc_ucontext_req req;
658 struct nes_alloc_ucontext_resp uresp;
659 struct nes_ucontext *nes_ucontext;
660 struct nes_ib_device *nesibdev = nesvnic->nesibdev;
661
662
663 if (ib_copy_from_udata(&req, udata, sizeof(struct nes_alloc_ucontext_req))) {
664 printk(KERN_ERR PFX "Invalid structure size on allocate user context.\n");
665 return ERR_PTR(-EINVAL);
666 }
667
668 if (req.userspace_ver != NES_ABI_USERSPACE_VER) {
669 printk(KERN_ERR PFX "Invalid userspace driver version detected. Detected version %d, should be %d\n",
670 req.userspace_ver, NES_ABI_USERSPACE_VER);
671 return ERR_PTR(-EINVAL);
672 }
673
674
675 memset(&uresp, 0, sizeof uresp);
676
677 uresp.max_qps = nesibdev->max_qp;
678 uresp.max_pds = nesibdev->max_pd;
679 uresp.wq_size = nesdev->nesadapter->max_qp_wr * 2;
680 uresp.virtwq = nesadapter->virtwq;
681 uresp.kernel_ver = NES_ABI_KERNEL_VER;
682
683 nes_ucontext = kzalloc(sizeof *nes_ucontext, GFP_KERNEL);
684 if (!nes_ucontext)
685 return ERR_PTR(-ENOMEM);
686
687 nes_ucontext->nesdev = nesdev;
688 nes_ucontext->mmap_wq_offset = uresp.max_pds;
689 nes_ucontext->mmap_cq_offset = nes_ucontext->mmap_wq_offset +
690 ((sizeof(struct nes_hw_qp_wqe) * uresp.max_qps * 2) + PAGE_SIZE-1) /
691 PAGE_SIZE;
692
693
694 if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) {
695 kfree(nes_ucontext);
696 return ERR_PTR(-EFAULT);
697 }
698
699 INIT_LIST_HEAD(&nes_ucontext->cq_reg_mem_list);
700 INIT_LIST_HEAD(&nes_ucontext->qp_reg_mem_list);
701 atomic_set(&nes_ucontext->usecnt, 1);
702 return &nes_ucontext->ibucontext;
703}
704
705
706/**
707 * nes_dealloc_ucontext
708 */
709static int nes_dealloc_ucontext(struct ib_ucontext *context)
710{
711 /* struct nes_vnic *nesvnic = to_nesvnic(context->device); */
712 /* struct nes_device *nesdev = nesvnic->nesdev; */
713 struct nes_ucontext *nes_ucontext = to_nesucontext(context);
714
715 if (!atomic_dec_and_test(&nes_ucontext->usecnt))
716 return 0;
717 kfree(nes_ucontext);
718 return 0;
719}
720
721
722/**
723 * nes_mmap
724 */
725static int nes_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
726{
727 unsigned long index;
728 struct nes_vnic *nesvnic = to_nesvnic(context->device);
729 struct nes_device *nesdev = nesvnic->nesdev;
730 /* struct nes_adapter *nesadapter = nesdev->nesadapter; */
731 struct nes_ucontext *nes_ucontext;
732 struct nes_qp *nesqp;
733
734 nes_ucontext = to_nesucontext(context);
735
736
737 if (vma->vm_pgoff >= nes_ucontext->mmap_wq_offset) {
738 index = (vma->vm_pgoff - nes_ucontext->mmap_wq_offset) * PAGE_SIZE;
739 index /= ((sizeof(struct nes_hw_qp_wqe) * nesdev->nesadapter->max_qp_wr * 2) +
740 PAGE_SIZE-1) & (~(PAGE_SIZE-1));
741 if (!test_bit(index, nes_ucontext->allocated_wqs)) {
742 nes_debug(NES_DBG_MMAP, "wq %lu not allocated\n", index);
743 return -EFAULT;
744 }
745 nesqp = nes_ucontext->mmap_nesqp[index];
746 if (nesqp == NULL) {
747 nes_debug(NES_DBG_MMAP, "wq %lu has a NULL QP base.\n", index);
748 return -EFAULT;
749 }
750 if (remap_pfn_range(vma, vma->vm_start,
751 virt_to_phys(nesqp->hwqp.sq_vbase) >> PAGE_SHIFT,
752 vma->vm_end - vma->vm_start,
753 vma->vm_page_prot)) {
754 nes_debug(NES_DBG_MMAP, "remap_pfn_range failed.\n");
755 return -EAGAIN;
756 }
757 vma->vm_private_data = nesqp;
758 return 0;
759 } else {
760 index = vma->vm_pgoff;
761 if (!test_bit(index, nes_ucontext->allocated_doorbells))
762 return -EFAULT;
763
764 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
765 if (io_remap_pfn_range(vma, vma->vm_start,
766 (nesdev->doorbell_start +
767 ((nes_ucontext->mmap_db_index[index] - nesdev->base_doorbell_index) * 4096))
768 >> PAGE_SHIFT, PAGE_SIZE, vma->vm_page_prot))
769 return -EAGAIN;
770 vma->vm_private_data = nes_ucontext;
771 return 0;
772 }
773
774 return -ENOSYS;
775}
776
777
778/**
779 * nes_alloc_pd
780 */
781static struct ib_pd *nes_alloc_pd(struct ib_device *ibdev,
782 struct ib_ucontext *context, struct ib_udata *udata)
783{
784 struct nes_pd *nespd;
785 struct nes_vnic *nesvnic = to_nesvnic(ibdev);
786 struct nes_device *nesdev = nesvnic->nesdev;
787 struct nes_adapter *nesadapter = nesdev->nesadapter;
788 struct nes_ucontext *nesucontext;
789 struct nes_alloc_pd_resp uresp;
790 u32 pd_num = 0;
791 int err;
792
793 nes_debug(NES_DBG_PD, "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n",
794 nesvnic, nesdev->netdev[0], nesdev->netdev[0]->name, ibdev, context,
29b4433d 795 netdev_refcnt_read(nesvnic->netdev));
3c2d774c
GS		 796
797 err = nes_alloc_resource(nesadapter, nesadapter->allocated_pds,
81821644 798 nesadapter->max_pd, &pd_num, &nesadapter->next_pd, NES_RESOURCE_PD);
3c2d774c
GS		 799 if (err) {
800 return ERR_PTR(err);
801 }
802
803 nespd = kzalloc(sizeof (struct nes_pd), GFP_KERNEL);
804 if (!nespd) {
805 nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num);
806 return ERR_PTR(-ENOMEM);
807 }
808
809 nes_debug(NES_DBG_PD, "Allocating PD (%p) for ib device %s\n",
810 nespd, nesvnic->nesibdev->ibdev.name);
811
812 nespd->pd_id = (pd_num << (PAGE_SHIFT-12)) + nesadapter->base_pd;
813
814 if (context) {
815 nesucontext = to_nesucontext(context);
816 nespd->mmap_db_index = find_next_zero_bit(nesucontext->allocated_doorbells,
817 NES_MAX_USER_DB_REGIONS, nesucontext->first_free_db);
818 nes_debug(NES_DBG_PD, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n",
819 nespd->mmap_db_index, nespd->pd_id);
65b07ec2 820 if (nespd->mmap_db_index >= NES_MAX_USER_DB_REGIONS) {
3c2d774c
GS		 821 nes_debug(NES_DBG_PD, "mmap_db_index > MAX\n");
822 nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num);
823 kfree(nespd);
824 return ERR_PTR(-ENOMEM);
825 }
826
827 uresp.pd_id = nespd->pd_id;
828 uresp.mmap_db_index = nespd->mmap_db_index;
829 if (ib_copy_to_udata(udata, &uresp, sizeof (struct nes_alloc_pd_resp))) {
830 nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num);
831 kfree(nespd);
832 return ERR_PTR(-EFAULT);
833 }
834
835 set_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells);
836 nesucontext->mmap_db_index[nespd->mmap_db_index] = nespd->pd_id;
837 nesucontext->first_free_db = nespd->mmap_db_index + 1;
838 }
839
840 nes_debug(NES_DBG_PD, "PD%u structure located @%p.\n", nespd->pd_id, nespd);
841 return &nespd->ibpd;
842}
843
844
845/**
846 * nes_dealloc_pd
847 */
848static int nes_dealloc_pd(struct ib_pd *ibpd)
849{
850 struct nes_ucontext *nesucontext;
851 struct nes_pd *nespd = to_nespd(ibpd);
852 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
853 struct nes_device *nesdev = nesvnic->nesdev;
854 struct nes_adapter *nesadapter = nesdev->nesadapter;
855
856 if ((ibpd->uobject) && (ibpd->uobject->context)) {
857 nesucontext = to_nesucontext(ibpd->uobject->context);
858 nes_debug(NES_DBG_PD, "Clearing bit %u from allocated doorbells\n",
859 nespd->mmap_db_index);
860 clear_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells);
861 nesucontext->mmap_db_index[nespd->mmap_db_index] = 0;
862 if (nesucontext->first_free_db > nespd->mmap_db_index) {
863 nesucontext->first_free_db = nespd->mmap_db_index;
864 }
865 }
866
867 nes_debug(NES_DBG_PD, "Deallocating PD%u structure located @%p.\n",
868 nespd->pd_id, nespd);
869 nes_free_resource(nesadapter, nesadapter->allocated_pds,
870 (nespd->pd_id-nesadapter->base_pd)>>(PAGE_SHIFT-12));
871 kfree(nespd);
872
873 return 0;
874}
875
876
877/**
878 * nes_create_ah
879 */
880static struct ib_ah *nes_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
881{
882 return ERR_PTR(-ENOSYS);
883}
884
885
886/**
887 * nes_destroy_ah
888 */
889static int nes_destroy_ah(struct ib_ah *ah)
890{
891 return -ENOSYS;
892}
893
894
895/**
896 * nes_get_encoded_size
897 */
898static inline u8 nes_get_encoded_size(int *size)
899{
900 u8 encoded_size = 0;
901 if (*size <= 32) {
902 *size = 32;
903 encoded_size = 1;
904 } else if (*size <= 128) {
905 *size = 128;
906 encoded_size = 2;
907 } else if (*size <= 512) {
908 *size = 512;
909 encoded_size = 3;
910 }
911 return (encoded_size);
912}
913
914
915
916/**
917 * nes_setup_virt_qp
918 */
919static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl,
920 struct nes_vnic *nesvnic, int sq_size, int rq_size)
921{
922 unsigned long flags;
923 void *mem;
924 __le64 *pbl = NULL;
925 __le64 *tpbl;
926 __le64 *pblbuffer;
927 struct nes_device *nesdev = nesvnic->nesdev;
928 struct nes_adapter *nesadapter = nesdev->nesadapter;
929 u32 pbl_entries;
930 u8 rq_pbl_entries;
931 u8 sq_pbl_entries;
932
933 pbl_entries = nespbl->pbl_size >> 3;
71e0957c 934 nes_debug(NES_DBG_QP, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n",
3c2d774c
GS		 935 nespbl->pbl_size, pbl_entries,
936 (void *)nespbl->pbl_vbase,
71e0957c 937 (unsigned long) nespbl->pbl_pbase);
3c2d774c
GS		 938 pbl = (__le64 *) nespbl->pbl_vbase; /* points to first pbl entry */
939 /* now lets set the sq_vbase as well as rq_vbase addrs we will assign */
940 /* the first pbl to be fro the rq_vbase... */
941 rq_pbl_entries = (rq_size * sizeof(struct nes_hw_qp_wqe)) >> 12;
942 sq_pbl_entries = (sq_size * sizeof(struct nes_hw_qp_wqe)) >> 12;
943 nesqp->hwqp.sq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32);
944 if (!nespbl->page) {
945 nes_debug(NES_DBG_QP, "QP nespbl->page is NULL \n");
946 kfree(nespbl);
947 return -ENOMEM;
948 }
949
950 nesqp->hwqp.sq_vbase = kmap(nespbl->page);
951 nesqp->page = nespbl->page;
952 if (!nesqp->hwqp.sq_vbase) {
953 nes_debug(NES_DBG_QP, "QP sq_vbase kmap failed\n");
954 kfree(nespbl);
955 return -ENOMEM;
956 }
957
958 /* Now to get to sq.. we need to calculate how many */
959 /* PBL entries were used by the rq.. */
960 pbl += sq_pbl_entries;
961 nesqp->hwqp.rq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32);
962 /* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */
963 /*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */
964
71e0957c
RD		 965 nes_debug(NES_DBG_QP, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n",
966 nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase,
967 nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase);
3c2d774c
GS		 968 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
969 if (!nesadapter->free_256pbl) {
970 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
971 nespbl->pbl_pbase);
972 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
973 kunmap(nesqp->page);
974 kfree(nespbl);
975 return -ENOMEM;
976 }
977 nesadapter->free_256pbl--;
978 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
979
980 nesqp->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 256, &nesqp->pbl_pbase);
981 pblbuffer = nesqp->pbl_vbase;
982 if (!nesqp->pbl_vbase) {
983 /* memory allocated during nes_reg_user_mr() */
984 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
985 nespbl->pbl_pbase);
986 kfree(nespbl);
987 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
988 nesadapter->free_256pbl++;
989 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
990 kunmap(nesqp->page);
991 return -ENOMEM;
992 }
993 memset(nesqp->pbl_vbase, 0, 256);
994 /* fill in the page address in the pbl buffer.. */
995 tpbl = pblbuffer + 16;
996 pbl = (__le64 *)nespbl->pbl_vbase;
997 while (sq_pbl_entries--)
998 *tpbl++ = *pbl++;
999 tpbl = pblbuffer;
1000 while (rq_pbl_entries--)
1001 *tpbl++ = *pbl++;
1002
1003 /* done with memory allocated during nes_reg_user_mr() */
1004 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
1005 nespbl->pbl_pbase);
1006 kfree(nespbl);
1007
1008 nesqp->qp_mem_size =
1009 max((u32)sizeof(struct nes_qp_context), ((u32)256)) + 256; /* this is Q2 */
1010 /* Round up to a multiple of a page */
1011 nesqp->qp_mem_size += PAGE_SIZE - 1;
1012 nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);
1013
1014 mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,
1015 &nesqp->hwqp.q2_pbase);
1016
1017 if (!mem) {
1018 pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase);
1019 nesqp->pbl_vbase = NULL;
1020 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
1021 nesadapter->free_256pbl++;
1022 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
1023 kunmap(nesqp->page);
1024 return -ENOMEM;
1025 }
d2fa9b26 1026 nesqp->sq_kmapped = 1;
3c2d774c
GS		 1027 nesqp->hwqp.q2_vbase = mem;
1028 mem += 256;
1029 memset(nesqp->hwqp.q2_vbase, 0, 256);
1030 nesqp->nesqp_context = mem;
1031 memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));
1032 nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;
1033
1034 return 0;
1035}
1036
1037
1038/**
1039 * nes_setup_mmap_qp
1040 */
1041static int nes_setup_mmap_qp(struct nes_qp *nesqp, struct nes_vnic *nesvnic,
1042 int sq_size, int rq_size)
1043{
1044 void *mem;
1045 struct nes_device *nesdev = nesvnic->nesdev;
1046
1047 nesqp->qp_mem_size = (sizeof(struct nes_hw_qp_wqe) * sq_size) +
1048 (sizeof(struct nes_hw_qp_wqe) * rq_size) +
1049 max((u32)sizeof(struct nes_qp_context), ((u32)256)) +
1050 256; /* this is Q2 */
1051 /* Round up to a multiple of a page */
1052 nesqp->qp_mem_size += PAGE_SIZE - 1;
1053 nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);
1054
1055 mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,
1056 &nesqp->hwqp.sq_pbase);
1057 if (!mem)
1058 return -ENOMEM;
1059 nes_debug(NES_DBG_QP, "PCI consistent memory for "
1060 "host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n",
1061 mem, (unsigned long)nesqp->hwqp.sq_pbase, nesqp->qp_mem_size);
1062
1063 memset(mem, 0, nesqp->qp_mem_size);
1064
1065 nesqp->hwqp.sq_vbase = mem;
1066 mem += sizeof(struct nes_hw_qp_wqe) * sq_size;
1067
1068 nesqp->hwqp.rq_vbase = mem;
1069 nesqp->hwqp.rq_pbase = nesqp->hwqp.sq_pbase +
1070 sizeof(struct nes_hw_qp_wqe) * sq_size;
1071 mem += sizeof(struct nes_hw_qp_wqe) * rq_size;
1072
1073 nesqp->hwqp.q2_vbase = mem;
1074 nesqp->hwqp.q2_pbase = nesqp->hwqp.rq_pbase +
1075 sizeof(struct nes_hw_qp_wqe) * rq_size;
1076 mem += 256;
1077 memset(nesqp->hwqp.q2_vbase, 0, 256);
1078
1079 nesqp->nesqp_context = mem;
1080 nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;
1081 memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));
1082 return 0;
1083}
1084
1085
1086/**
1087 * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory.
1088 */
1089static inline void nes_free_qp_mem(struct nes_device *nesdev,
1090 struct nes_qp *nesqp, int virt_wqs)
1091{
1092 unsigned long flags;
1093 struct nes_adapter *nesadapter = nesdev->nesadapter;
1094 if (!virt_wqs) {
1095 pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size,
1096 nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase);
1097 }else {
1098 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
1099 nesadapter->free_256pbl++;
1100 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
1101 pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase);
1102 pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase );
1103 nesqp->pbl_vbase = NULL;
d2fa9b26
FL		 1104 if (nesqp->sq_kmapped) {
1105 nesqp->sq_kmapped = 0;
1106 kunmap(nesqp->page);
1107 }
3c2d774c
GS		 1108 }
1109}
1110
1111
1112/**
1113 * nes_create_qp
1114 */
1115static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
1116 struct ib_qp_init_attr *init_attr, struct ib_udata *udata)
1117{
1118 u64 u64temp= 0;
1119 u64 u64nesqp = 0;
1120 struct nes_pd *nespd = to_nespd(ibpd);
1121 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
1122 struct nes_device *nesdev = nesvnic->nesdev;
1123 struct nes_adapter *nesadapter = nesdev->nesadapter;
1124 struct nes_qp *nesqp;
1125 struct nes_cq *nescq;
1126 struct nes_ucontext *nes_ucontext;
1127 struct nes_hw_cqp_wqe *cqp_wqe;
1128 struct nes_cqp_request *cqp_request;
1129 struct nes_create_qp_req req;
1130 struct nes_create_qp_resp uresp;
1131 struct nes_pbl *nespbl = NULL;
1132 u32 qp_num = 0;
1133 u32 opcode = 0;
1134 /* u32 counter = 0; */
1135 void *mem;
1136 unsigned long flags;
1137 int ret;
1138 int err;
1139 int virt_wqs = 0;
1140 int sq_size;
1141 int rq_size;
1142 u8 sq_encoded_size;
1143 u8 rq_encoded_size;
1144 /* int counter; */
1145
b846f25a
EC		 1146 if (init_attr->create_flags)
1147 return ERR_PTR(-EINVAL);
1148
3c2d774c
GS		 1149 atomic_inc(&qps_created);
1150 switch (init_attr->qp_type) {
1151 case IB_QPT_RC:
1152 if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) {
1153 init_attr->cap.max_inline_data = 0;
1154 } else {
1155 init_attr->cap.max_inline_data = 64;
1156 }
1157 sq_size = init_attr->cap.max_send_wr;
1158 rq_size = init_attr->cap.max_recv_wr;
1159
7495ab68 1160 /* check if the encoded sizes are OK or not... */
3c2d774c
GS		 1161 sq_encoded_size = nes_get_encoded_size(&sq_size);
1162 rq_encoded_size = nes_get_encoded_size(&rq_size);
1163
1164 if ((!sq_encoded_size) || (!rq_encoded_size)) {
1165 nes_debug(NES_DBG_QP, "ERROR bad rq (%u) or sq (%u) size\n",
1166 rq_size, sq_size);
1167 return ERR_PTR(-EINVAL);
1168 }
1169
1170 init_attr->cap.max_send_wr = sq_size -2;
1171 init_attr->cap.max_recv_wr = rq_size -1;
1172 nes_debug(NES_DBG_QP, "RQ size=%u, SQ Size=%u\n", rq_size, sq_size);
1173
1174 ret = nes_alloc_resource(nesadapter, nesadapter->allocated_qps,
81821644 1175 nesadapter->max_qp, &qp_num, &nesadapter->next_qp, NES_RESOURCE_QP);
3c2d774c
GS		 1176 if (ret) {
1177 return ERR_PTR(ret);
1178 }
1179
1180 /* Need 512 (actually now 1024) byte alignment on this structure */
1181 mem = kzalloc(sizeof(*nesqp)+NES_SW_CONTEXT_ALIGN-1, GFP_KERNEL);
1182 if (!mem) {
1183 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
1184 nes_debug(NES_DBG_QP, "Unable to allocate QP\n");
1185 return ERR_PTR(-ENOMEM);
1186 }
1187 u64nesqp = (unsigned long)mem;
1188 u64nesqp += ((u64)NES_SW_CONTEXT_ALIGN) - 1;
1189 u64temp = ((u64)NES_SW_CONTEXT_ALIGN) - 1;
1190 u64nesqp &= ~u64temp;
1191 nesqp = (struct nes_qp *)(unsigned long)u64nesqp;
1192 /* nes_debug(NES_DBG_QP, "nesqp=%p, allocated buffer=%p. Rounded to closest %u\n",
1193 nesqp, mem, NES_SW_CONTEXT_ALIGN); */
1194 nesqp->allocated_buffer = mem;
1195
1196 if (udata) {
1197 if (ib_copy_from_udata(&req, udata, sizeof(struct nes_create_qp_req))) {
1198 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
1199 kfree(nesqp->allocated_buffer);
1200 nes_debug(NES_DBG_QP, "ib_copy_from_udata() Failed \n");
9d194d10 1201 return ERR_PTR(-EFAULT);
3c2d774c
GS		 1202 }
1203 if (req.user_wqe_buffers) {
1204 virt_wqs = 1;
1205 }
43adff39
TN		 1206 if (req.user_qp_buffer)
1207 nesqp->nesuqp_addr = req.user_qp_buffer;
3c2d774c
GS		 1208 if ((ibpd->uobject) && (ibpd->uobject->context)) {
1209 nesqp->user_mode = 1;
1210 nes_ucontext = to_nesucontext(ibpd->uobject->context);
1211 if (virt_wqs) {
1212 err = 1;
1213 list_for_each_entry(nespbl, &nes_ucontext->qp_reg_mem_list, list) {
1214 if (nespbl->user_base == (unsigned long )req.user_wqe_buffers) {
1215 list_del(&nespbl->list);
1216 err = 0;
1217 nes_debug(NES_DBG_QP, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n",
1218 nespbl, nespbl->user_base);
1219 break;
1220 }
1221 }
1222 if (err) {
1223 nes_debug(NES_DBG_QP, "Didn't Find PBL for virtual QP. address = %llx.\n",
1224 (long long unsigned int)req.user_wqe_buffers);
1225 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
1226 kfree(nesqp->allocated_buffer);
9300c0c0 1227 return ERR_PTR(-EFAULT);
3c2d774c
GS		 1228 }
1229 }
1230
1231 nes_ucontext = to_nesucontext(ibpd->uobject->context);
1232 nesqp->mmap_sq_db_index =
1233 find_next_zero_bit(nes_ucontext->allocated_wqs,
1234 NES_MAX_USER_WQ_REGIONS, nes_ucontext->first_free_wq);
1235 /* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n",
1236 nespd->mmap_db_index); */
51af33e8 1237 if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) {
3c2d774c
GS		 1238 nes_debug(NES_DBG_QP,
1239 "db index > max user regions, failing create QP\n");
1240 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
1241 if (virt_wqs) {
1242 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
1243 nespbl->pbl_pbase);
1244 kfree(nespbl);
1245 }
1246 kfree(nesqp->allocated_buffer);
1247 return ERR_PTR(-ENOMEM);
1248 }
1249 set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);
1250 nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp;
1251 nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1;
1252 } else {
1253 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
1254 kfree(nesqp->allocated_buffer);
1255 return ERR_PTR(-EFAULT);
1256 }
1257 }
1258 err = (!virt_wqs) ? nes_setup_mmap_qp(nesqp, nesvnic, sq_size, rq_size) :
1259 nes_setup_virt_qp(nesqp, nespbl, nesvnic, sq_size, rq_size);
1260 if (err) {
1261 nes_debug(NES_DBG_QP,
1262 "error geting qp mem code = %d\n", err);
1263 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
1264 kfree(nesqp->allocated_buffer);
1265 return ERR_PTR(-ENOMEM);
1266 }
1267
1268 nesqp->hwqp.sq_size = sq_size;
1269 nesqp->hwqp.sq_encoded_size = sq_encoded_size;
1270 nesqp->hwqp.sq_head = 1;
1271 nesqp->hwqp.rq_size = rq_size;
1272 nesqp->hwqp.rq_encoded_size = rq_encoded_size;
1273 /* nes_debug(NES_DBG_QP, "nesqp->nesqp_context_pbase = %p\n",
1274 (void *)nesqp->nesqp_context_pbase);
1275 */
1276 nesqp->hwqp.qp_id = qp_num;
1277 nesqp->ibqp.qp_num = nesqp->hwqp.qp_id;
1278 nesqp->nespd = nespd;
1279
1280 nescq = to_nescq(init_attr->send_cq);
1281 nesqp->nesscq = nescq;
1282 nescq = to_nescq(init_attr->recv_cq);
1283 nesqp->nesrcq = nescq;
1284
1285 nesqp->nesqp_context->misc |= cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) <<
1286 NES_QPCONTEXT_MISC_PCI_FCN_SHIFT);
1287 nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.rq_encoded_size <<
1288 NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT);
1289 nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.sq_encoded_size <<
1290 NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT);
c12e56ef 1291 if (!udata) {
3c2d774c
GS		 1292 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN);
1293 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN);
c12e56ef 1294 }
3c2d774c
GS		 1295 nesqp->nesqp_context->cqs = cpu_to_le32(nesqp->nesscq->hw_cq.cq_number +
1296 ((u32)nesqp->nesrcq->hw_cq.cq_number << 16));
1297 u64temp = (u64)nesqp->hwqp.sq_pbase;
1298 nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp);
1299 nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32));
1300
1301
1302 if (!virt_wqs) {
1303 u64temp = (u64)nesqp->hwqp.sq_pbase;
1304 nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp);
1305 nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32));
1306 u64temp = (u64)nesqp->hwqp.rq_pbase;
1307 nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp);
1308 nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32));
1309 } else {
1310 u64temp = (u64)nesqp->pbl_pbase;
1311 nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp);
1312 nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32));
1313 }
1314
1315 /* nes_debug(NES_DBG_QP, "next_qp_nic_index=%u, using nic_index=%d\n",
1316 nesvnic->next_qp_nic_index,
1317 nesvnic->qp_nic_index[nesvnic->next_qp_nic_index]); */
1318 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1319 nesqp->nesqp_context->misc2 |= cpu_to_le32(
1320 (u32)nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] <<
1321 NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT);
1322 nesvnic->next_qp_nic_index++;
1323 if ((nesvnic->next_qp_nic_index > 3) ||
1324 (nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] == 0xf)) {
1325 nesvnic->next_qp_nic_index = 0;
1326 }
1327 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1328
1329 nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32((u32)nesqp->nespd->pd_id << 16);
1330 u64temp = (u64)nesqp->hwqp.q2_pbase;
1331 nesqp->nesqp_context->q2_addr_low = cpu_to_le32((u32)u64temp);
1332 nesqp->nesqp_context->q2_addr_high = cpu_to_le32((u32)(u64temp >> 32));
1333 nesqp->nesqp_context->aeq_token_low = cpu_to_le32((u32)((unsigned long)(nesqp)));
1334 nesqp->nesqp_context->aeq_token_high = cpu_to_le32((u32)(upper_32_bits((unsigned long)(nesqp))));
1335 nesqp->nesqp_context->ird_ord_sizes = cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM |
883c6992 1336 NES_QPCONTEXT_ORDIRD_AAH |
3c2d774c
GS		 1337 ((((u32)nesadapter->max_irrq_wr) <<
1338 NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT) & NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK));
1339 if (disable_mpa_crc) {
1340 nes_debug(NES_DBG_QP, "Disabling MPA crc checking due to module option.\n");
1341 nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC);
1342 }
1343
1344
1345 /* Create the QP */
1346 cqp_request = nes_get_cqp_request(nesdev);
1347 if (cqp_request == NULL) {
1348 nes_debug(NES_DBG_QP, "Failed to get a cqp_request\n");
1349 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
1350 nes_free_qp_mem(nesdev, nesqp,virt_wqs);
1351 kfree(nesqp->allocated_buffer);
1352 return ERR_PTR(-ENOMEM);
1353 }
1354 cqp_request->waiting = 1;
1355 cqp_wqe = &cqp_request->cqp_wqe;
1356
1357 if (!virt_wqs) {
1358 opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP |
1359 NES_CQP_QP_IWARP_STATE_IDLE;
1360 } else {
1361 opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_VIRT_WQS |
1362 NES_CQP_QP_IWARP_STATE_IDLE;
1363 }
1364 opcode |= NES_CQP_QP_CQS_VALID;
1365 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1366 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
1367 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);
1368
1369 u64temp = (u64)nesqp->nesqp_context_pbase;
1370 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1371
1372 atomic_set(&cqp_request->refcount, 2);
8294f297 1373 nes_post_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 1374
1375 /* Wait for CQP */
1376 nes_debug(NES_DBG_QP, "Waiting for create iWARP QP%u to complete.\n",
1377 nesqp->hwqp.qp_id);
1378 ret = wait_event_timeout(cqp_request->waitq,
1379 (cqp_request->request_done != 0), NES_EVENT_TIMEOUT);
1380 nes_debug(NES_DBG_QP, "Create iwarp QP%u completed, wait_event_timeout ret=%u,"
1381 " nesdev->cqp_head = %u, nesdev->cqp.sq_tail = %u,"
1382 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
1383 nesqp->hwqp.qp_id, ret, nesdev->cqp.sq_head, nesdev->cqp.sq_tail,
1384 cqp_request->major_code, cqp_request->minor_code);
1385 if ((!ret) || (cqp_request->major_code)) {
1ff66e8c 1386 nes_put_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 1387 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
1388 nes_free_qp_mem(nesdev, nesqp,virt_wqs);
1389 kfree(nesqp->allocated_buffer);
1390 if (!ret) {
1391 return ERR_PTR(-ETIME);
1392 } else {
1393 return ERR_PTR(-EIO);
1394 }
3c2d774c
GS		 1395 }
1396
1ff66e8c
RD		 1397 nes_put_cqp_request(nesdev, cqp_request);
1398
3c2d774c
GS		 1399 if (ibpd->uobject) {
1400 uresp.mmap_sq_db_index = nesqp->mmap_sq_db_index;
60429630 1401 uresp.mmap_rq_db_index = 0;
3c2d774c
GS		 1402 uresp.actual_sq_size = sq_size;
1403 uresp.actual_rq_size = rq_size;
1404 uresp.qp_id = nesqp->hwqp.qp_id;
1405 uresp.nes_drv_opt = nes_drv_opt;
1406 if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) {
1407 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
1408 nes_free_qp_mem(nesdev, nesqp,virt_wqs);
1409 kfree(nesqp->allocated_buffer);
1410 return ERR_PTR(-EFAULT);
1411 }
1412 }
1413
1414 nes_debug(NES_DBG_QP, "QP%u structure located @%p.Size = %u.\n",
1415 nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp));
1416 spin_lock_init(&nesqp->lock);
3c2d774c
GS		 1417 nes_add_ref(&nesqp->ibqp);
1418 break;
1419 default:
1420 nes_debug(NES_DBG_QP, "Invalid QP type: %d\n", init_attr->qp_type);
1421 return ERR_PTR(-EINVAL);
3c2d774c
GS		 1422 }
1423
d14152da 1424 nesqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR);
7bfcfa51
TN		 1425 init_timer(&nesqp->terminate_timer);
1426 nesqp->terminate_timer.function = nes_terminate_timeout;
1427 nesqp->terminate_timer.data = (unsigned long)nesqp;
d14152da 1428
3c2d774c
GS		 1429 /* update the QP table */
1430 nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp;
1431 nes_debug(NES_DBG_QP, "netdev refcnt=%u\n",
29b4433d 1432 netdev_refcnt_read(nesvnic->netdev));
3c2d774c
GS		 1433
1434 return &nesqp->ibqp;
1435}
1436
5ee21fe0
DW		 1437/**
1438 * nes_clean_cq
1439 */
1440static void nes_clean_cq(struct nes_qp *nesqp, struct nes_cq *nescq)
1441{
1442 u32 cq_head;
1443 u32 lo;
1444 u32 hi;
1445 u64 u64temp;
1446 unsigned long flags = 0;
1447
1448 spin_lock_irqsave(&nescq->lock, flags);
1449
1450 cq_head = nescq->hw_cq.cq_head;
1451 while (le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
1452 rmb();
1453 lo = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
1454 hi = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]);
1455 u64temp = (((u64)hi) << 32) | ((u64)lo);
1456 u64temp &= ~(NES_SW_CONTEXT_ALIGN-1);
1457 if (u64temp == (u64)(unsigned long)nesqp) {
1458 /* Zero the context value so cqe will be ignored */
1459 nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 0;
1460 nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX] = 0;
1461 }
1462
1463 if (++cq_head >= nescq->hw_cq.cq_size)
1464 cq_head = 0;
1465 }
1466
1467 spin_unlock_irqrestore(&nescq->lock, flags);
1468}
1469
1470
3c2d774c
GS		 1471/**
1472 * nes_destroy_qp
1473 */
1474static int nes_destroy_qp(struct ib_qp *ibqp)
1475{
1476 struct nes_qp *nesqp = to_nesqp(ibqp);
3c2d774c
GS		 1477 struct nes_ucontext *nes_ucontext;
1478 struct ib_qp_attr attr;
1479 struct iw_cm_id *cm_id;
1480 struct iw_cm_event cm_event;
0f0bee8b 1481 int ret = 0;
3c2d774c
GS		 1482
1483 atomic_inc(&sw_qps_destroyed);
1484 nesqp->destroyed = 1;
1485
1486 /* Blow away the connection if it exists. */
1487 if (nesqp->ibqp_state >= IB_QPS_INIT && nesqp->ibqp_state <= IB_QPS_RTS) {
1488 /* if (nesqp->ibqp_state == IB_QPS_RTS) { */
1489 attr.qp_state = IB_QPS_ERR;
1490 nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL);
1491 }
1492
1493 if (((nesqp->ibqp_state == IB_QPS_INIT) ||
1494 (nesqp->ibqp_state == IB_QPS_RTR)) && (nesqp->cm_id)) {
1495 cm_id = nesqp->cm_id;
1496 cm_event.event = IW_CM_EVENT_CONNECT_REPLY;
d0c49bf3 1497 cm_event.status = -ETIMEDOUT;
3c2d774c
GS		 1498 cm_event.local_addr = cm_id->local_addr;
1499 cm_event.remote_addr = cm_id->remote_addr;
1500 cm_event.private_data = NULL;
1501 cm_event.private_data_len = 0;
1502
1503 nes_debug(NES_DBG_QP, "Generating a CM Timeout Event for "
1504 "QP%u. cm_id = %p, refcount = %u. \n",
1505 nesqp->hwqp.qp_id, cm_id, atomic_read(&nesqp->refcount));
1506
1507 cm_id->rem_ref(cm_id);
1508 ret = cm_id->event_handler(cm_id, &cm_event);
1509 if (ret)
1510 nes_debug(NES_DBG_QP, "OFA CM event_handler returned, ret=%d\n", ret);
1511 }
1512
3c2d774c
GS		 1513 if (nesqp->user_mode) {
1514 if ((ibqp->uobject)&&(ibqp->uobject->context)) {
1515 nes_ucontext = to_nesucontext(ibqp->uobject->context);
1516 clear_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);
1517 nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = NULL;
1518 if (nes_ucontext->first_free_wq > nesqp->mmap_sq_db_index) {
1519 nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index;
1520 }
1521 }
d2fa9b26
FL		 1522 if (nesqp->pbl_pbase && nesqp->sq_kmapped) {
1523 nesqp->sq_kmapped = 0;
3c2d774c 1524 kunmap(nesqp->page);
d2fa9b26 1525 }
5ee21fe0
DW		 1526 } else {
1527 /* Clean any pending completions from the cq(s) */
1528 if (nesqp->nesscq)
1529 nes_clean_cq(nesqp, nesqp->nesscq);
1530
1531 if ((nesqp->nesrcq) && (nesqp->nesrcq != nesqp->nesscq))
1532 nes_clean_cq(nesqp, nesqp->nesrcq);
3c2d774c 1533 }
3c2d774c
GS		 1534 nes_rem_ref(&nesqp->ibqp);
1535 return 0;
1536}
1537
1538
1539/**
1540 * nes_create_cq
1541 */
bcf4c1ea
MB		 1542static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
1543 const struct ib_cq_init_attr *attr,
1544 struct ib_ucontext *context,
1545 struct ib_udata *udata)
3c2d774c 1546{
bcf4c1ea 1547 int entries = attr->cqe;
3c2d774c
GS		 1548 u64 u64temp;
1549 struct nes_vnic *nesvnic = to_nesvnic(ibdev);
1550 struct nes_device *nesdev = nesvnic->nesdev;
1551 struct nes_adapter *nesadapter = nesdev->nesadapter;
1552 struct nes_cq *nescq;
1553 struct nes_ucontext *nes_ucontext = NULL;
1554 struct nes_cqp_request *cqp_request;
1555 void *mem = NULL;
1556 struct nes_hw_cqp_wqe *cqp_wqe;
1557 struct nes_pbl *nespbl = NULL;
1558 struct nes_create_cq_req req;
1559 struct nes_create_cq_resp resp;
1560 u32 cq_num = 0;
1561 u32 opcode = 0;
1562 u32 pbl_entries = 1;
1563 int err;
1564 unsigned long flags;
1565 int ret;
1566
bcf4c1ea
MB		 1567 if (attr->flags)
1568 return ERR_PTR(-EINVAL);
1569
5924aea6
CT		 1570 if (entries > nesadapter->max_cqe)
1571 return ERR_PTR(-EINVAL);
1572
3c2d774c 1573 err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs,
81821644 1574 nesadapter->max_cq, &cq_num, &nesadapter->next_cq, NES_RESOURCE_CQ);
3c2d774c
GS		 1575 if (err) {
1576 return ERR_PTR(err);
1577 }
1578
1579 nescq = kzalloc(sizeof(struct nes_cq), GFP_KERNEL);
1580 if (!nescq) {
1581 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1582 nes_debug(NES_DBG_CQ, "Unable to allocate nes_cq struct\n");
1583 return ERR_PTR(-ENOMEM);
1584 }
1585
1586 nescq->hw_cq.cq_size = max(entries + 1, 5);
1587 nescq->hw_cq.cq_number = cq_num;
1588 nescq->ibcq.cqe = nescq->hw_cq.cq_size - 1;
1589
1590
1591 if (context) {
1592 nes_ucontext = to_nesucontext(context);
1593 if (ib_copy_from_udata(&req, udata, sizeof (struct nes_create_cq_req))) {
1594 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1595 kfree(nescq);
1596 return ERR_PTR(-EFAULT);
1597 }
1598 nesvnic->mcrq_ucontext = nes_ucontext;
1599 nes_ucontext->mcrqf = req.mcrqf;
1600 if (nes_ucontext->mcrqf) {
1601 if (nes_ucontext->mcrqf & 0x80000000)
2e369544 1602 nescq->hw_cq.cq_number = nesvnic->nic.qp_id + 28 + 2 * ((nes_ucontext->mcrqf & 0xf) - 1);
3c2d774c
GS		 1603 else if (nes_ucontext->mcrqf & 0x40000000)
1604 nescq->hw_cq.cq_number = nes_ucontext->mcrqf & 0xffff;
1605 else
1606 nescq->hw_cq.cq_number = nesvnic->mcrq_qp_id + nes_ucontext->mcrqf-1;
5d1af5c8 1607 nescq->mcrqf = nes_ucontext->mcrqf;
3c2d774c
GS		 1608 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1609 }
1610 nes_debug(NES_DBG_CQ, "CQ Virtual Address = %08lX, size = %u.\n",
1611 (unsigned long)req.user_cq_buffer, entries);
9300c0c0 1612 err = 1;
3c2d774c
GS		 1613 list_for_each_entry(nespbl, &nes_ucontext->cq_reg_mem_list, list) {
1614 if (nespbl->user_base == (unsigned long )req.user_cq_buffer) {
1615 list_del(&nespbl->list);
1616 err = 0;
1617 nes_debug(NES_DBG_CQ, "Found PBL for virtual CQ. nespbl=%p.\n",
1618 nespbl);
1619 break;
1620 }
1621 }
1622 if (err) {
1623 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1624 kfree(nescq);
9300c0c0 1625 return ERR_PTR(-EFAULT);
3c2d774c
GS		 1626 }
1627
1628 pbl_entries = nespbl->pbl_size >> 3;
1629 nescq->cq_mem_size = 0;
1630 } else {
1631 nescq->cq_mem_size = nescq->hw_cq.cq_size * sizeof(struct nes_hw_cqe);
1632 nes_debug(NES_DBG_CQ, "Attempting to allocate pci memory (%u entries, %u bytes) for CQ%u.\n",
1633 entries, nescq->cq_mem_size, nescq->hw_cq.cq_number);
1634
1635 /* allocate the physical buffer space */
9011a67b
JP		 1636 mem = pci_zalloc_consistent(nesdev->pcidev, nescq->cq_mem_size,
1637 &nescq->hw_cq.cq_pbase);
3c2d774c
GS		 1638 if (!mem) {
1639 printk(KERN_ERR PFX "Unable to allocate pci memory for cq\n");
1640 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1641 kfree(nescq);
1642 return ERR_PTR(-ENOMEM);
1643 }
1644
3c2d774c
GS		 1645 nescq->hw_cq.cq_vbase = mem;
1646 nescq->hw_cq.cq_head = 0;
1647 nes_debug(NES_DBG_CQ, "CQ%u virtual address @ %p, phys = 0x%08X\n",
1648 nescq->hw_cq.cq_number, nescq->hw_cq.cq_vbase,
1649 (u32)nescq->hw_cq.cq_pbase);
1650 }
1651
1652 nescq->hw_cq.ce_handler = nes_iwarp_ce_handler;
1653 spin_lock_init(&nescq->lock);
1654
1655 /* send CreateCQ request to CQP */
1656 cqp_request = nes_get_cqp_request(nesdev);
1657 if (cqp_request == NULL) {
1658 nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n");
1659 if (!context)
1660 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
1661 nescq->hw_cq.cq_pbase);
5d1af5c8
MW		 1662 else {
1663 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size,
1664 nespbl->pbl_vbase, nespbl->pbl_pbase);
1665 kfree(nespbl);
1666 }
1667
3c2d774c
GS		 1668 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1669 kfree(nescq);
1670 return ERR_PTR(-ENOMEM);
1671 }
1672 cqp_request->waiting = 1;
1673 cqp_wqe = &cqp_request->cqp_wqe;
1674
1675 opcode = NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1676 NES_CQP_CQ_CHK_OVERFLOW |
1677 NES_CQP_CQ_CEQE_MASK | ((u32)nescq->hw_cq.cq_size << 16);
1678
1679 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
1680
1681 if (pbl_entries != 1) {
1682 if (pbl_entries > 32) {
1683 /* use 4k pbl */
1684 nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 4k PBL\n", pbl_entries);
1685 if (nesadapter->free_4kpbl == 0) {
c5d321e5
FL		 1686 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
1687 nes_free_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 1688 if (!context)
1689 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
1690 nescq->hw_cq.cq_pbase);
5d1af5c8
MW		 1691 else {
1692 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size,
1693 nespbl->pbl_vbase, nespbl->pbl_pbase);
1694 kfree(nespbl);
1695 }
3c2d774c
GS		 1696 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1697 kfree(nescq);
1698 return ERR_PTR(-ENOMEM);
1699 } else {
1700 opcode |= (NES_CQP_CQ_VIRT | NES_CQP_CQ_4KB_CHUNK);
1701 nescq->virtual_cq = 2;
1702 nesadapter->free_4kpbl--;
1703 }
1704 } else {
1705 /* use 256 byte pbl */
1706 nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 256 byte PBL\n", pbl_entries);
1707 if (nesadapter->free_256pbl == 0) {
c5d321e5
FL		 1708 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
1709 nes_free_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 1710 if (!context)
1711 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
1712 nescq->hw_cq.cq_pbase);
5d1af5c8
MW		 1713 else {
1714 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size,
1715 nespbl->pbl_vbase, nespbl->pbl_pbase);
1716 kfree(nespbl);
1717 }
3c2d774c
GS		 1718 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1719 kfree(nescq);
1720 return ERR_PTR(-ENOMEM);
1721 } else {
1722 opcode |= NES_CQP_CQ_VIRT;
1723 nescq->virtual_cq = 1;
1724 nesadapter->free_256pbl--;
1725 }
1726 }
1727 }
1728
1729 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
1730
1731 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1732 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
1733 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1734 (nescq->hw_cq.cq_number | ((u32)nesdev->ceq_index << 16)));
1735
1736 if (context) {
1737 if (pbl_entries != 1)
1738 u64temp = (u64)nespbl->pbl_pbase;
1739 else
1740 u64temp = le64_to_cpu(nespbl->pbl_vbase[0]);
1741 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX,
1742 nes_ucontext->mmap_db_index[0]);
1743 } else {
1744 u64temp = (u64)nescq->hw_cq.cq_pbase;
1745 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1746 }
1747 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1748 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1749 u64temp = (u64)(unsigned long)&nescq->hw_cq;
1750 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1751 cpu_to_le32((u32)(u64temp >> 1));
1752 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1753 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1754
1755 atomic_set(&cqp_request->refcount, 2);
8294f297 1756 nes_post_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 1757
1758 /* Wait for CQP */
1759 nes_debug(NES_DBG_CQ, "Waiting for create iWARP CQ%u to complete.\n",
1760 nescq->hw_cq.cq_number);
1761 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done),
1762 NES_EVENT_TIMEOUT * 2);
1763 nes_debug(NES_DBG_CQ, "Create iWARP CQ%u completed, wait_event_timeout ret = %d.\n",
1764 nescq->hw_cq.cq_number, ret);
1765 if ((!ret) || (cqp_request->major_code)) {
1ff66e8c 1766 nes_put_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 1767 if (!context)
1768 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
1769 nescq->hw_cq.cq_pbase);
5d1af5c8
MW		 1770 else {
1771 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size,
1772 nespbl->pbl_vbase, nespbl->pbl_pbase);
1773 kfree(nespbl);
1774 }
3c2d774c
GS		 1775 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1776 kfree(nescq);
1777 return ERR_PTR(-EIO);
3c2d774c 1778 }
1ff66e8c 1779 nes_put_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 1780
1781 if (context) {
1782 /* free the nespbl */
1783 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
1784 nespbl->pbl_pbase);
1785 kfree(nespbl);
1786 resp.cq_id = nescq->hw_cq.cq_number;
1787 resp.cq_size = nescq->hw_cq.cq_size;
1788 resp.mmap_db_index = 0;
60429630 1789 if (ib_copy_to_udata(udata, &resp, sizeof resp - sizeof resp.reserved)) {
3c2d774c
GS		 1790 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
1791 kfree(nescq);
1792 return ERR_PTR(-EFAULT);
1793 }
1794 }
1795
1796 return &nescq->ibcq;
1797}
1798
1799
1800/**
1801 * nes_destroy_cq
1802 */
1803static int nes_destroy_cq(struct ib_cq *ib_cq)
1804{
1805 struct nes_cq *nescq;
1806 struct nes_device *nesdev;
1807 struct nes_vnic *nesvnic;
1808 struct nes_adapter *nesadapter;
1809 struct nes_hw_cqp_wqe *cqp_wqe;
1810 struct nes_cqp_request *cqp_request;
1811 unsigned long flags;
1812 u32 opcode = 0;
1813 int ret;
1814
1815 if (ib_cq == NULL)
1816 return 0;
1817
1818 nescq = to_nescq(ib_cq);
1819 nesvnic = to_nesvnic(ib_cq->device);
1820 nesdev = nesvnic->nesdev;
1821 nesadapter = nesdev->nesadapter;
1822
1823 nes_debug(NES_DBG_CQ, "Destroy CQ%u\n", nescq->hw_cq.cq_number);
1824
1825 /* Send DestroyCQ request to CQP */
1826 cqp_request = nes_get_cqp_request(nesdev);
1827 if (cqp_request == NULL) {
1828 nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n");
1829 return -ENOMEM;
1830 }
1831 cqp_request->waiting = 1;
1832 cqp_wqe = &cqp_request->cqp_wqe;
1833 opcode = NES_CQP_DESTROY_CQ | (nescq->hw_cq.cq_size << 16);
1834 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
1835 if (nescq->virtual_cq == 1) {
1836 nesadapter->free_256pbl++;
1837 if (nesadapter->free_256pbl > nesadapter->max_256pbl) {
1838 printk(KERN_ERR PFX "%s: free 256B PBLs(%u) has exceeded the max(%u)\n",
33718363 1839 __func__, nesadapter->free_256pbl, nesadapter->max_256pbl);
3c2d774c
GS		 1840 }
1841 } else if (nescq->virtual_cq == 2) {
1842 nesadapter->free_4kpbl++;
1843 if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) {
1844 printk(KERN_ERR PFX "%s: free 4K PBLs(%u) has exceeded the max(%u)\n",
33718363 1845 __func__, nesadapter->free_4kpbl, nesadapter->max_4kpbl);
3c2d774c
GS		 1846 }
1847 opcode |= NES_CQP_CQ_4KB_CHUNK;
1848 }
1849
1850 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
1851
1852 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1853 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
1854 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1855 (nescq->hw_cq.cq_number | ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 16)));
5d1af5c8
MW		 1856 if (!nescq->mcrqf)
1857 nes_free_resource(nesadapter, nesadapter->allocated_cqs, nescq->hw_cq.cq_number);
1858
3c2d774c 1859 atomic_set(&cqp_request->refcount, 2);
8294f297 1860 nes_post_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 1861
1862 /* Wait for CQP */
1863 nes_debug(NES_DBG_CQ, "Waiting for destroy iWARP CQ%u to complete.\n",
1864 nescq->hw_cq.cq_number);
1865 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done),
1866 NES_EVENT_TIMEOUT);
1867 nes_debug(NES_DBG_CQ, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u,"
1868 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
1869 nescq->hw_cq.cq_number, ret, cqp_request->major_code,
1870 cqp_request->minor_code);
1ff66e8c
RD		 1871 if (!ret) {
1872 nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy timeout expired\n",
3c2d774c 1873 nescq->hw_cq.cq_number);
1ff66e8c
RD		 1874 ret = -ETIME;
1875 } else if (cqp_request->major_code) {
1876 nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy failed\n",
3c2d774c 1877 nescq->hw_cq.cq_number);
1ff66e8c 1878 ret = -EIO;
3c2d774c
GS		 1879 } else {
1880 ret = 0;
3c2d774c 1881 }
1ff66e8c 1882 nes_put_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 1883
1884 if (nescq->cq_mem_size)
1885 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size,
31d1e340 1886 nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase);
3c2d774c
GS		 1887 kfree(nescq);
1888
1889 return ret;
1890}
1891
0145f341
DW		 1892/**
1893 * root_256
1894 */
1895static u32 root_256(struct nes_device *nesdev,
1896 struct nes_root_vpbl *root_vpbl,
1897 struct nes_root_vpbl *new_root,
cc005fa2 1898 u16 pbl_count_4k)
0145f341
DW		 1899{
1900 u64 leaf_pbl;
1901 int i, j, k;
1902
1903 if (pbl_count_4k == 1) {
1904 new_root->pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
1905 512, &new_root->pbl_pbase);
1906
1907 if (new_root->pbl_vbase == NULL)
1908 return 0;
1909
1910 leaf_pbl = (u64)root_vpbl->pbl_pbase;
1911 for (i = 0; i < 16; i++) {
1912 new_root->pbl_vbase[i].pa_low =
1913 cpu_to_le32((u32)leaf_pbl);
1914 new_root->pbl_vbase[i].pa_high =
1915 cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
1916 leaf_pbl += 256;
1917 }
1918 } else {
1919 for (i = 3; i >= 0; i--) {
1920 j = i * 16;
1921 root_vpbl->pbl_vbase[j] = root_vpbl->pbl_vbase[i];
1922 leaf_pbl = le32_to_cpu(root_vpbl->pbl_vbase[j].pa_low) +
1923 (((u64)le32_to_cpu(root_vpbl->pbl_vbase[j].pa_high))
1924 << 32);
1925 for (k = 1; k < 16; k++) {
1926 leaf_pbl += 256;
1927 root_vpbl->pbl_vbase[j + k].pa_low =
1928 cpu_to_le32((u32)leaf_pbl);
1929 root_vpbl->pbl_vbase[j + k].pa_high =
1930 cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
1931 }
1932 }
1933 }
1934
1935 return 1;
1936}
1937
3c2d774c
GS		 1938
1939/**
1940 * nes_reg_mr
1941 */
1942static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
1943 u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl,
0145f341
DW		 1944 dma_addr_t single_buffer, u16 pbl_count_4k,
1945 u16 residual_page_count_4k, int acc, u64 *iova_start,
1946 u16 *actual_pbl_cnt, u8 *used_4k_pbls)
3c2d774c
GS		 1947{
1948 struct nes_hw_cqp_wqe *cqp_wqe;
1949 struct nes_cqp_request *cqp_request;
1950 unsigned long flags;
1951 int ret;
1952 struct nes_adapter *nesadapter = nesdev->nesadapter;
0145f341 1953 uint pg_cnt = 0;
1af9222b 1954 u16 pbl_count_256 = 0;
0145f341
DW		 1955 u16 pbl_count = 0;
1956 u8 use_256_pbls = 0;
1957 u8 use_4k_pbls = 0;
1958 u16 use_two_level = (pbl_count_4k > 1) ? 1 : 0;
812d8672 1959 struct nes_root_vpbl new_root = { 0, NULL, NULL };
3c2d774c
GS		 1960 u32 opcode = 0;
1961 u16 major_code;
1962
1963 /* Register the region with the adapter */
1964 cqp_request = nes_get_cqp_request(nesdev);
1965 if (cqp_request == NULL) {
1966 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
1967 return -ENOMEM;
1968 }
1969 cqp_request->waiting = 1;
1970 cqp_wqe = &cqp_request->cqp_wqe;
1971
0145f341
DW		 1972 if (pbl_count_4k) {
1973 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
1974
1975 pg_cnt = ((pbl_count_4k - 1) * 512) + residual_page_count_4k;
1976 pbl_count_256 = (pg_cnt + 31) / 32;
1977 if (pg_cnt <= 32) {
1978 if (pbl_count_256 <= nesadapter->free_256pbl)
1979 use_256_pbls = 1;
1980 else if (pbl_count_4k <= nesadapter->free_4kpbl)
1981 use_4k_pbls = 1;
1982 } else if (pg_cnt <= 2048) {
1983 if (((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) &&
1984 (nesadapter->free_4kpbl > (nesadapter->max_4kpbl >> 1))) {
1985 use_4k_pbls = 1;
1986 } else if ((pbl_count_256 + 1) <= nesadapter->free_256pbl) {
1987 use_256_pbls = 1;
1988 use_two_level = 1;
1989 } else if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
1990 use_4k_pbls = 1;
3c2d774c
GS		 1991 }
1992 } else {
0145f341
DW		 1993 if ((pbl_count_4k + 1) <= nesadapter->free_4kpbl)
1994 use_4k_pbls = 1;
3c2d774c 1995 }
0145f341
DW		 1996
1997 if (use_256_pbls) {
1998 pbl_count = pbl_count_256;
1999 nesadapter->free_256pbl -= pbl_count + use_two_level;
2000 } else if (use_4k_pbls) {
2001 pbl_count = pbl_count_4k;
2002 nesadapter->free_4kpbl -= pbl_count + use_two_level;
2003 } else {
2004 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
2005 nes_debug(NES_DBG_MR, "Out of Pbls\n");
2006 nes_free_cqp_request(nesdev, cqp_request);
2007 return -ENOMEM;
2008 }
2009
2010 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
3c2d774c
GS		 2011 }
2012
0145f341 2013 if (use_256_pbls && use_two_level) {
cc005fa2 2014 if (root_256(nesdev, root_vpbl, &new_root, pbl_count_4k) == 1) {
0145f341
DW		 2015 if (new_root.pbl_pbase != 0)
2016 root_vpbl = &new_root;
2017 } else {
2018 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
2019 nesadapter->free_256pbl += pbl_count_256 + use_two_level;
2020 use_256_pbls = 0;
2021
2022 if (pbl_count_4k == 1)
2023 use_two_level = 0;
2024 pbl_count = pbl_count_4k;
2025
2026 if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
2027 nesadapter->free_4kpbl -= pbl_count + use_two_level;
2028 use_4k_pbls = 1;
2029 }
2030 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
2031
2032 if (use_4k_pbls == 0)
2033 return -ENOMEM;
2034 }
2035 }
3c2d774c
GS		 2036
2037 opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ |
2038 NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR;
2039 if (acc & IB_ACCESS_LOCAL_WRITE)
2040 opcode |= NES_CQP_STAG_RIGHTS_LOCAL_WRITE;
2041 if (acc & IB_ACCESS_REMOTE_WRITE)
2042 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_REM_ACC_EN;
2043 if (acc & IB_ACCESS_REMOTE_READ)
2044 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_READ | NES_CQP_STAG_REM_ACC_EN;
2045 if (acc & IB_ACCESS_MW_BIND)
2046 opcode |= NES_CQP_STAG_RIGHTS_WINDOW_BIND | NES_CQP_STAG_REM_ACC_EN;
2047
2048 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2049 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
2050 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, *iova_start);
2051 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, region_length);
2052
2053 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] =
2054 cpu_to_le32((u32)(region_length >> 8) & 0xff000000);
2055 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |=
2056 cpu_to_le32(nespd->pd_id & 0x00007fff);
2057 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
2058
2059 if (pbl_count == 0) {
2060 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, single_buffer);
2061 } else {
2062 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase);
2063 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count);
0145f341 2064 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (pg_cnt * 8));
3c2d774c 2065
0145f341 2066 if (use_4k_pbls)
3c2d774c
GS		 2067 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
2068 }
2069 barrier();
2070
2071 atomic_set(&cqp_request->refcount, 2);
8294f297 2072 nes_post_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 2073
2074 /* Wait for CQP */
2075 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done),
2076 NES_EVENT_TIMEOUT);
2077 nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
2078 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
2079 stag, ret, cqp_request->major_code, cqp_request->minor_code);
2080 major_code = cqp_request->major_code;
1ff66e8c
RD		 2081 nes_put_cqp_request(nesdev, cqp_request);
2082
dae5d13a
DW		 2083 if ((!ret || major_code) && pbl_count != 0) {
2084 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
0145f341
DW		 2085 if (use_256_pbls)
2086 nesadapter->free_256pbl += pbl_count + use_two_level;
2087 else if (use_4k_pbls)
2088 nesadapter->free_4kpbl += pbl_count + use_two_level;
dae5d13a
DW		 2089 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
2090 }
0145f341
DW		 2091 if (new_root.pbl_pbase)
2092 pci_free_consistent(nesdev->pcidev, 512, new_root.pbl_vbase,
2093 new_root.pbl_pbase);
2094
3c2d774c
GS		 2095 if (!ret)
2096 return -ETIME;
2097 else if (major_code)
2098 return -EIO;
3c2d774c 2099
0145f341
DW		 2100 *actual_pbl_cnt = pbl_count + use_two_level;
2101 *used_4k_pbls = use_4k_pbls;
3c2d774c
GS		 2102 return 0;
2103}
2104
2105
2106/**
2107 * nes_reg_phys_mr
2108 */
2109static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
2110 struct ib_phys_buf *buffer_list, int num_phys_buf, int acc,
2111 u64 * iova_start)
2112{
2113 u64 region_length;
2114 struct nes_pd *nespd = to_nespd(ib_pd);
2115 struct nes_vnic *nesvnic = to_nesvnic(ib_pd->device);
2116 struct nes_device *nesdev = nesvnic->nesdev;
2117 struct nes_adapter *nesadapter = nesdev->nesadapter;
2118 struct nes_mr *nesmr;
2119 struct ib_mr *ibmr;
2120 struct nes_vpbl vpbl;
2121 struct nes_root_vpbl root_vpbl;
2122 u32 stag;
2123 u32 i;
3f32eb11 2124 unsigned long mask;
3c2d774c
GS		 2125 u32 stag_index = 0;
2126 u32 next_stag_index = 0;
2127 u32 driver_key = 0;
2128 u32 root_pbl_index = 0;
2129 u32 cur_pbl_index = 0;
69d51023 2130 int err = 0;
3c2d774c
GS		 2131 int ret = 0;
2132 u16 pbl_count = 0;
2133 u8 single_page = 1;
2134 u8 stag_key = 0;
2135
3c2d774c
GS		 2136 region_length = 0;
2137 vpbl.pbl_vbase = NULL;
2138 root_vpbl.pbl_vbase = NULL;
2139 root_vpbl.pbl_pbase = 0;
2140
2141 get_random_bytes(&next_stag_index, sizeof(next_stag_index));
2142 stag_key = (u8)next_stag_index;
2143
2144 driver_key = 0;
2145
2146 next_stag_index >>= 8;
2147 next_stag_index %= nesadapter->max_mr;
2148 if (num_phys_buf > (1024*512)) {
2149 return ERR_PTR(-E2BIG);
2150 }
2151
3f32eb11
DW		 2152 if ((buffer_list[0].addr ^ *iova_start) & ~PAGE_MASK)
2153 return ERR_PTR(-EINVAL);
2154
3c2d774c 2155 err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, nesadapter->max_mr,
81821644 2156 &stag_index, &next_stag_index, NES_RESOURCE_PHYS_MR);
3c2d774c
GS		 2157 if (err) {
2158 return ERR_PTR(err);
2159 }
2160
2161 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
2162 if (!nesmr) {
2163 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2164 return ERR_PTR(-ENOMEM);
2165 }
2166
2167 for (i = 0; i < num_phys_buf; i++) {
2168
2169 if ((i & 0x01FF) == 0) {
2170 if (root_pbl_index == 1) {
2171 /* Allocate the root PBL */
2172 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 8192,
2173 &root_vpbl.pbl_pbase);
2174 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2175 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
2176 if (!root_vpbl.pbl_vbase) {
2177 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2178 vpbl.pbl_pbase);
2179 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2180 kfree(nesmr);
2181 return ERR_PTR(-ENOMEM);
2182 }
2183 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024, GFP_KERNEL);
2184 if (!root_vpbl.leaf_vpbl) {
2185 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
2186 root_vpbl.pbl_pbase);
2187 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2188 vpbl.pbl_pbase);
2189 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2190 kfree(nesmr);
2191 return ERR_PTR(-ENOMEM);
2192 }
2193 root_vpbl.pbl_vbase[0].pa_low = cpu_to_le32((u32)vpbl.pbl_pbase);
2194 root_vpbl.pbl_vbase[0].pa_high =
2195 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
2196 root_vpbl.leaf_vpbl[0] = vpbl;
2197 }
2198 /* Allocate a 4K buffer for the PBL */
2199 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
2200 &vpbl.pbl_pbase);
2201 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n",
2202 vpbl.pbl_vbase, (unsigned long)vpbl.pbl_pbase);
2203 if (!vpbl.pbl_vbase) {
2204 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2205 ibmr = ERR_PTR(-ENOMEM);
2206 kfree(nesmr);
2207 goto reg_phys_err;
2208 }
2209 /* Fill in the root table */
2210 if (1 <= root_pbl_index) {
2211 root_vpbl.pbl_vbase[root_pbl_index].pa_low =
2212 cpu_to_le32((u32)vpbl.pbl_pbase);
2213 root_vpbl.pbl_vbase[root_pbl_index].pa_high =
2214 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
2215 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
2216 }
2217 root_pbl_index++;
2218 cur_pbl_index = 0;
2219 }
3c2d774c 2220
3f32eb11
DW		 2221 mask = !buffer_list[i].size;
2222 if (i != 0)
2223 mask |= buffer_list[i].addr;
2224 if (i != num_phys_buf - 1)
2225 mask |= buffer_list[i].addr + buffer_list[i].size;
2226
2227 if (mask & ~PAGE_MASK) {
3c2d774c 2228 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
3f32eb11 2229 nes_debug(NES_DBG_MR, "Invalid buffer addr or size\n");
3c2d774c
GS		 2230 ibmr = ERR_PTR(-EINVAL);
2231 kfree(nesmr);
2232 goto reg_phys_err;
2233 }
2234
2235 region_length += buffer_list[i].size;
2236 if ((i != 0) && (single_page)) {
2237 if ((buffer_list[i-1].addr+PAGE_SIZE) != buffer_list[i].addr)
2238 single_page = 0;
2239 }
3f32eb11 2240 vpbl.pbl_vbase[cur_pbl_index].pa_low = cpu_to_le32((u32)buffer_list[i].addr & PAGE_MASK);
3c2d774c
GS		 2241 vpbl.pbl_vbase[cur_pbl_index++].pa_high =
2242 cpu_to_le32((u32)((((u64)buffer_list[i].addr) >> 32)));
2243 }
2244
2245 stag = stag_index << 8;
2246 stag |= driver_key;
2247 stag += (u32)stag_key;
2248
2249 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%016lX,"
2250 " length = 0x%016lX, index = 0x%08X\n",
2251 stag, (unsigned long)*iova_start, (unsigned long)region_length, stag_index);
2252
3c2d774c
GS		 2253 /* Make the leaf PBL the root if only one PBL */
2254 if (root_pbl_index == 1) {
2255 root_vpbl.pbl_pbase = vpbl.pbl_pbase;
2256 }
2257
2258 if (single_page) {
2259 pbl_count = 0;
2260 } else {
2261 pbl_count = root_pbl_index;
2262 }
2263 ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl,
0145f341
DW		 2264 buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start,
2265 &nesmr->pbls_used, &nesmr->pbl_4k);
3c2d774c
GS		 2266
2267 if (ret == 0) {
2268 nesmr->ibmr.rkey = stag;
2269 nesmr->ibmr.lkey = stag;
2270 nesmr->mode = IWNES_MEMREG_TYPE_MEM;
2271 ibmr = &nesmr->ibmr;
3c2d774c
GS		 2272 } else {
2273 kfree(nesmr);
2274 ibmr = ERR_PTR(-ENOMEM);
2275 }
2276
2277 reg_phys_err:
2278 /* free the resources */
2279 if (root_pbl_index == 1) {
2280 /* single PBL case */
2281 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, vpbl.pbl_pbase);
2282 } else {
2283 for (i=0; i<root_pbl_index; i++) {
2284 pci_free_consistent(nesdev->pcidev, 4096, root_vpbl.leaf_vpbl[i].pbl_vbase,
2285 root_vpbl.leaf_vpbl[i].pbl_pbase);
2286 }
2287 kfree(root_vpbl.leaf_vpbl);
2288 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
2289 root_vpbl.pbl_pbase);
2290 }
2291
2292 return ibmr;
2293}
2294
2295
2296/**
2297 * nes_get_dma_mr
2298 */
2299static struct ib_mr *nes_get_dma_mr(struct ib_pd *pd, int acc)
2300{
2301 struct ib_phys_buf bl;
2302 u64 kva = 0;
2303
2304 nes_debug(NES_DBG_MR, "\n");
2305
2306 bl.size = (u64)0xffffffffffULL;
2307 bl.addr = 0;
2308 return nes_reg_phys_mr(pd, &bl, 1, acc, &kva);
2309}
2310
2311
2312/**
2313 * nes_reg_user_mr
2314 */
2315static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2316 u64 virt, int acc, struct ib_udata *udata)
2317{
2318 u64 iova_start;
2319 __le64 *pbl;
2320 u64 region_length;
2321 dma_addr_t last_dma_addr = 0;
2322 dma_addr_t first_dma_addr = 0;
2323 struct nes_pd *nespd = to_nespd(pd);
2324 struct nes_vnic *nesvnic = to_nesvnic(pd->device);
2325 struct nes_device *nesdev = nesvnic->nesdev;
2326 struct nes_adapter *nesadapter = nesdev->nesadapter;
2327 struct ib_mr *ibmr = ERR_PTR(-EINVAL);
eeb8461e 2328 struct scatterlist *sg;
3c2d774c
GS		 2329 struct nes_ucontext *nes_ucontext;
2330 struct nes_pbl *nespbl;
2331 struct nes_mr *nesmr;
2332 struct ib_umem *region;
2333 struct nes_mem_reg_req req;
2334 struct nes_vpbl vpbl;
2335 struct nes_root_vpbl root_vpbl;
eeb8461e 2336 int entry, page_index;
3c2d774c
GS		 2337 int page_count = 0;
2338 int err, pbl_depth = 0;
2339 int chunk_pages;
2340 int ret;
2341 u32 stag;
2342 u32 stag_index = 0;
2343 u32 next_stag_index;
2344 u32 driver_key;
2345 u32 root_pbl_index = 0;
2346 u32 cur_pbl_index = 0;
2347 u32 skip_pages;
2348 u16 pbl_count;
2349 u8 single_page = 1;
2350 u8 stag_key;
eeb8461e 2351 int first_page = 1;
3c2d774c 2352
cb9fbc5c 2353 region = ib_umem_get(pd->uobject->context, start, length, acc, 0);
3c2d774c
GS		 2354 if (IS_ERR(region)) {
2355 return (struct ib_mr *)region;
2356 }
2357
2358 nes_debug(NES_DBG_MR, "User base = 0x%lX, Virt base = 0x%lX, length = %u,"
2359 " offset = %u, page size = %u.\n",
2360 (unsigned long int)start, (unsigned long int)virt, (u32)length,
406f9e5f 2361 ib_umem_offset(region), region->page_size);
3c2d774c 2362
406f9e5f 2363 skip_pages = ((u32)ib_umem_offset(region)) >> 12;
3c2d774c 2364
e2e435f2
JL		 2365 if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2366 ib_umem_release(region);
3c2d774c 2367 return ERR_PTR(-EFAULT);
e2e435f2 2368 }
3c2d774c
GS		 2369 nes_debug(NES_DBG_MR, "Memory Registration type = %08X.\n", req.reg_type);
2370
2371 switch (req.reg_type) {
2372 case IWNES_MEMREG_TYPE_MEM:
2373 pbl_depth = 0;
2374 region_length = 0;
2375 vpbl.pbl_vbase = NULL;
2376 root_vpbl.pbl_vbase = NULL;
2377 root_vpbl.pbl_pbase = 0;
2378
2379 get_random_bytes(&next_stag_index, sizeof(next_stag_index));
2380 stag_key = (u8)next_stag_index;
2381
2382 driver_key = next_stag_index & 0x70000000;
2383
2384 next_stag_index >>= 8;
2385 next_stag_index %= nesadapter->max_mr;
2386
2387 err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
81821644 2388 nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_USER_MR);
3c2d774c
GS		 2389 if (err) {
2390 ib_umem_release(region);
2391 return ERR_PTR(err);
2392 }
2393
2394 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
2395 if (!nesmr) {
2396 ib_umem_release(region);
2397 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2398 return ERR_PTR(-ENOMEM);
2399 }
2400 nesmr->region = region;
2401
eeb8461e
YH		 2402 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
2403 if (sg_dma_address(sg) & ~PAGE_MASK) {
2404 ib_umem_release(region);
2405 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2406 nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n",
2407 (unsigned int) sg_dma_address(sg));
2408 ibmr = ERR_PTR(-EINVAL);
2409 kfree(nesmr);
2410 goto reg_user_mr_err;
2411 }
3c2d774c 2412
eeb8461e
YH		 2413 if (!sg_dma_len(sg)) {
2414 ib_umem_release(region);
2415 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2416 stag_index);
2417 nes_debug(NES_DBG_MR, "Invalid Buffer Size\n");
2418 ibmr = ERR_PTR(-EINVAL);
2419 kfree(nesmr);
2420 goto reg_user_mr_err;
2421 }
3c2d774c 2422
eeb8461e
YH		 2423 region_length += sg_dma_len(sg);
2424 chunk_pages = sg_dma_len(sg) >> 12;
2425 region_length -= skip_pages << 12;
2426 for (page_index = skip_pages; page_index < chunk_pages; page_index++) {
2427 skip_pages = 0;
406f9e5f 2428 if ((page_count != 0) && (page_count << 12) - (ib_umem_offset(region) & (4096 - 1)) >= region->length)
eeb8461e
YH		 2429 goto enough_pages;
2430 if ((page_count&0x01FF) == 0) {
2431 if (page_count >= 1024 * 512) {
2432 ib_umem_release(region);
2433 nes_free_resource(nesadapter,
2434 nesadapter->allocated_mrs, stag_index);
2435 kfree(nesmr);
2436 ibmr = ERR_PTR(-E2BIG);
2437 goto reg_user_mr_err;
2438 }
2439 if (root_pbl_index == 1) {
2440 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
2441 8192, &root_vpbl.pbl_pbase);
2442 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2443 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
2444 if (!root_vpbl.pbl_vbase) {
3c2d774c 2445 ib_umem_release(region);
eeb8461e
YH		 2446 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2447 vpbl.pbl_pbase);
2448 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2449 stag_index);
3c2d774c 2450 kfree(nesmr);
eeb8461e 2451 ibmr = ERR_PTR(-ENOMEM);
3c2d774c
GS		 2452 goto reg_user_mr_err;
2453 }
eeb8461e
YH		 2454 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024,
2455 GFP_KERNEL);
2456 if (!root_vpbl.leaf_vpbl) {
3c2d774c 2457 ib_umem_release(region);
eeb8461e
YH		 2458 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
2459 root_vpbl.pbl_pbase);
2460 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2461 vpbl.pbl_pbase);
2462 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2463 stag_index);
3c2d774c 2464 kfree(nesmr);
eeb8461e 2465 ibmr = ERR_PTR(-ENOMEM);
3c2d774c
GS		 2466 goto reg_user_mr_err;
2467 }
eeb8461e
YH		 2468 root_vpbl.pbl_vbase[0].pa_low =
2469 cpu_to_le32((u32)vpbl.pbl_pbase);
2470 root_vpbl.pbl_vbase[0].pa_high =
2471 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
2472 root_vpbl.leaf_vpbl[0] = vpbl;
3c2d774c 2473 }
eeb8461e
YH		 2474 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
2475 &vpbl.pbl_pbase);
2476 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
2477 vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase);
2478 if (!vpbl.pbl_vbase) {
2479 ib_umem_release(region);
2480 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
2481 ibmr = ERR_PTR(-ENOMEM);
2482 kfree(nesmr);
2483 goto reg_user_mr_err;
2484 }
2485 if (1 <= root_pbl_index) {
2486 root_vpbl.pbl_vbase[root_pbl_index].pa_low =
2487 cpu_to_le32((u32)vpbl.pbl_pbase);
2488 root_vpbl.pbl_vbase[root_pbl_index].pa_high =
2489 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32)));
2490 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
2491 }
2492 root_pbl_index++;
2493 cur_pbl_index = 0;
2494 }
2495 if (single_page) {
2496 if (page_count != 0) {
2497 if ((last_dma_addr+4096) !=
2498 (sg_dma_address(sg)+
2499 (page_index*4096)))
2500 single_page = 0;
2501 last_dma_addr = sg_dma_address(sg)+
2502 (page_index*4096);
2503 } else {
2504 first_dma_addr = sg_dma_address(sg)+
2505 (page_index*4096);
2506 last_dma_addr = first_dma_addr;
3c2d774c 2507 }
3c2d774c 2508 }
eeb8461e
YH		 2509
2510 vpbl.pbl_vbase[cur_pbl_index].pa_low =
2511 cpu_to_le32((u32)(sg_dma_address(sg)+
2512 (page_index*4096)));
2513 vpbl.pbl_vbase[cur_pbl_index].pa_high =
2514 cpu_to_le32((u32)((((u64)(sg_dma_address(sg)+
2515 (page_index*4096))) >> 32)));
2516 cur_pbl_index++;
2517 page_count++;
3c2d774c
GS		 2518 }
2519 }
eeb8461e 2520
3c2d774c
GS		 2521 enough_pages:
2522 nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x,"
2523 " stag_key=0x%08x\n",
2524 stag_index, driver_key, stag_key);
2525 stag = stag_index << 8;
2526 stag |= driver_key;
2527 stag += (u32)stag_key;
3c2d774c
GS		 2528
2529 iova_start = virt;
2530 /* Make the leaf PBL the root if only one PBL */
2531 if (root_pbl_index == 1) {
2532 root_vpbl.pbl_pbase = vpbl.pbl_pbase;
2533 }
2534
2535 if (single_page) {
2536 pbl_count = 0;
2537 } else {
2538 pbl_count = root_pbl_index;
2539 first_dma_addr = 0;
2540 }
2541 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%08X, length = 0x%08X,"
2542 " index = 0x%08X, region->length=0x%08llx, pbl_count = %u\n",
2543 stag, (unsigned int)iova_start,
2544 (unsigned int)region_length, stag_index,
2545 (unsigned long long)region->length, pbl_count);
0145f341
DW		 2546 ret = nes_reg_mr(nesdev, nespd, stag, region->length, &root_vpbl,
2547 first_dma_addr, pbl_count, (u16)cur_pbl_index, acc,
2548 &iova_start, &nesmr->pbls_used, &nesmr->pbl_4k);
3c2d774c
GS		 2549
2550 nes_debug(NES_DBG_MR, "ret=%d\n", ret);
2551
2552 if (ret == 0) {
2553 nesmr->ibmr.rkey = stag;
2554 nesmr->ibmr.lkey = stag;
2555 nesmr->mode = IWNES_MEMREG_TYPE_MEM;
2556 ibmr = &nesmr->ibmr;
3c2d774c
GS		 2557 } else {
2558 ib_umem_release(region);
2559 kfree(nesmr);
2560 ibmr = ERR_PTR(-ENOMEM);
2561 }
2562
2563 reg_user_mr_err:
2564 /* free the resources */
2565 if (root_pbl_index == 1) {
2566 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
2567 vpbl.pbl_pbase);
2568 } else {
2569 for (page_index=0; page_index<root_pbl_index; page_index++) {
2570 pci_free_consistent(nesdev->pcidev, 4096,
2571 root_vpbl.leaf_vpbl[page_index].pbl_vbase,
2572 root_vpbl.leaf_vpbl[page_index].pbl_pbase);
2573 }
2574 kfree(root_vpbl.leaf_vpbl);
2575 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
2576 root_vpbl.pbl_pbase);
2577 }
2578
2579 nes_debug(NES_DBG_MR, "Leaving, ibmr=%p", ibmr);
2580
2581 return ibmr;
3c2d774c
GS		 2582 case IWNES_MEMREG_TYPE_QP:
2583 case IWNES_MEMREG_TYPE_CQ:
7d9c199a
TN		 2584 if (!region->length) {
2585 nes_debug(NES_DBG_MR, "Unable to register zero length region for CQ\n");
2586 ib_umem_release(region);
2587 return ERR_PTR(-EINVAL);
2588 }
3c2d774c
GS		 2589 nespbl = kzalloc(sizeof(*nespbl), GFP_KERNEL);
2590 if (!nespbl) {
2591 nes_debug(NES_DBG_MR, "Unable to allocate PBL\n");
2592 ib_umem_release(region);
2593 return ERR_PTR(-ENOMEM);
2594 }
2595 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
2596 if (!nesmr) {
2597 ib_umem_release(region);
2598 kfree(nespbl);
2599 nes_debug(NES_DBG_MR, "Unable to allocate nesmr\n");
2600 return ERR_PTR(-ENOMEM);
2601 }
2602 nesmr->region = region;
2603 nes_ucontext = to_nesucontext(pd->uobject->context);
2604 pbl_depth = region->length >> 12;
2605 pbl_depth += (region->length & (4096-1)) ? 1 : 0;
2606 nespbl->pbl_size = pbl_depth*sizeof(u64);
2607 if (req.reg_type == IWNES_MEMREG_TYPE_QP) {
2608 nes_debug(NES_DBG_MR, "Attempting to allocate QP PBL memory");
2609 } else {
2610 nes_debug(NES_DBG_MR, "Attempting to allocate CP PBL memory");
2611 }
2612
2613 nes_debug(NES_DBG_MR, " %u bytes, %u entries.\n",
2614 nespbl->pbl_size, pbl_depth);
2615 pbl = pci_alloc_consistent(nesdev->pcidev, nespbl->pbl_size,
2616 &nespbl->pbl_pbase);
2617 if (!pbl) {
2618 ib_umem_release(region);
2619 kfree(nesmr);
2620 kfree(nespbl);
2621 nes_debug(NES_DBG_MR, "Unable to allocate PBL memory\n");
2622 return ERR_PTR(-ENOMEM);
2623 }
2624
2625 nespbl->pbl_vbase = (u64 *)pbl;
2626 nespbl->user_base = start;
71e0957c 2627 nes_debug(NES_DBG_MR, "Allocated PBL memory, %u bytes, pbl_pbase=%lx,"
3c2d774c 2628 " pbl_vbase=%p user_base=0x%lx\n",
71e0957c
RD		 2629 nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase,
2630 (void *) nespbl->pbl_vbase, nespbl->user_base);
3c2d774c 2631
eeb8461e
YH		 2632 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
2633 chunk_pages = sg_dma_len(sg) >> 12;
2634 chunk_pages += (sg_dma_len(sg) & (4096-1)) ? 1 : 0;
2635 if (first_page) {
2636 nespbl->page = sg_page(sg);
2637 first_page = 0;
2638 }
2639
2640 for (page_index = 0; page_index < chunk_pages; page_index++) {
2641 ((__le32 *)pbl)[0] = cpu_to_le32((u32)
2642 (sg_dma_address(sg)+
2643 (page_index*4096)));
2644 ((__le32 *)pbl)[1] = cpu_to_le32(((u64)
2645 (sg_dma_address(sg)+
2646 (page_index*4096)))>>32);
2647 nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl,
2648 (unsigned long long)*pbl,
2649 le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0]));
2650 pbl++;
3c2d774c
GS		 2651 }
2652 }
eeb8461e 2653
3c2d774c
GS		 2654 if (req.reg_type == IWNES_MEMREG_TYPE_QP) {
2655 list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list);
2656 } else {
2657 list_add_tail(&nespbl->list, &nes_ucontext->cq_reg_mem_list);
2658 }
2659 nesmr->ibmr.rkey = -1;
2660 nesmr->ibmr.lkey = -1;
2661 nesmr->mode = req.reg_type;
2662 return &nesmr->ibmr;
3c2d774c
GS		 2663 }
2664
e2e435f2 2665 ib_umem_release(region);
3c2d774c
GS		 2666 return ERR_PTR(-ENOSYS);
2667}
2668
2669
2670/**
2671 * nes_dereg_mr
2672 */
2673static int nes_dereg_mr(struct ib_mr *ib_mr)
2674{
2675 struct nes_mr *nesmr = to_nesmr(ib_mr);
2676 struct nes_vnic *nesvnic = to_nesvnic(ib_mr->device);
2677 struct nes_device *nesdev = nesvnic->nesdev;
2678 struct nes_adapter *nesadapter = nesdev->nesadapter;
2679 struct nes_hw_cqp_wqe *cqp_wqe;
2680 struct nes_cqp_request *cqp_request;
2681 unsigned long flags;
2682 int ret;
2683 u16 major_code;
2684 u16 minor_code;
2685
2686 if (nesmr->region) {
2687 ib_umem_release(nesmr->region);
2688 }
2689 if (nesmr->mode != IWNES_MEMREG_TYPE_MEM) {
2690 kfree(nesmr);
2691 return 0;
2692 }
2693
2694 /* Deallocate the region with the adapter */
2695
2696 cqp_request = nes_get_cqp_request(nesdev);
2697 if (cqp_request == NULL) {
2698 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
2699 return -ENOMEM;
2700 }
2701 cqp_request->waiting = 1;
2702 cqp_wqe = &cqp_request->cqp_wqe;
2703
3c2d774c
GS		 2704 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2705 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2706 NES_CQP_DEALLOCATE_STAG | NES_CQP_STAG_VA_TO |
2707 NES_CQP_STAG_DEALLOC_PBLS | NES_CQP_STAG_MR);
2708 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ib_mr->rkey);
2709
2710 atomic_set(&cqp_request->refcount, 2);
8294f297 2711 nes_post_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 2712
2713 /* Wait for CQP */
2714 nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X completed\n", ib_mr->rkey);
2715 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
2716 NES_EVENT_TIMEOUT);
2717 nes_debug(NES_DBG_MR, "Deallocate STag 0x%08X completed, wait_event_timeout ret = %u,"
2718 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
2719 ib_mr->rkey, ret, cqp_request->major_code, cqp_request->minor_code);
2720
3c2d774c
GS		 2721 major_code = cqp_request->major_code;
2722 minor_code = cqp_request->minor_code;
1ff66e8c
RD		 2723
2724 nes_put_cqp_request(nesdev, cqp_request);
2725
3c2d774c
GS		 2726 if (!ret) {
2727 nes_debug(NES_DBG_MR, "Timeout waiting to destroy STag,"
2728 " ib_mr=%p, rkey = 0x%08X\n",
2729 ib_mr, ib_mr->rkey);
2730 return -ETIME;
2731 } else if (major_code) {
2732 nes_debug(NES_DBG_MR, "Error (0x%04X:0x%04X) while attempting"
2733 " to destroy STag, ib_mr=%p, rkey = 0x%08X\n",
2734 major_code, minor_code, ib_mr, ib_mr->rkey);
2735 return -EIO;
dae5d13a
DW		 2736 }
2737
2738 if (nesmr->pbls_used != 0) {
2739 spin_lock_irqsave(&nesadapter->pbl_lock, flags);
2740 if (nesmr->pbl_4k) {
2741 nesadapter->free_4kpbl += nesmr->pbls_used;
2742 if (nesadapter->free_4kpbl > nesadapter->max_4kpbl)
2743 printk(KERN_ERR PFX "free 4KB PBLs(%u) has "
2744 "exceeded the max(%u)\n",
2745 nesadapter->free_4kpbl,
2746 nesadapter->max_4kpbl);
2747 } else {
2748 nesadapter->free_256pbl += nesmr->pbls_used;
2749 if (nesadapter->free_256pbl > nesadapter->max_256pbl)
2750 printk(KERN_ERR PFX "free 256B PBLs(%u) has "
2751 "exceeded the max(%u)\n",
2752 nesadapter->free_256pbl,
2753 nesadapter->max_256pbl);
2754 }
2755 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
2756 }
2757 nes_free_resource(nesadapter, nesadapter->allocated_mrs,
2758 (ib_mr->rkey & 0x0fffff00) >> 8);
2759
2760 kfree(nesmr);
2761
2762 return 0;
3c2d774c
GS		 2763}
2764
2765
2766/**
2767 * show_rev
2768 */
f4e91eb4
TJ		 2769static ssize_t show_rev(struct device *dev, struct device_attribute *attr,
2770 char *buf)
3c2d774c
GS		 2771{
2772 struct nes_ib_device *nesibdev =
f4e91eb4 2773 container_of(dev, struct nes_ib_device, ibdev.dev);
3c2d774c
GS		 2774 struct nes_vnic *nesvnic = nesibdev->nesvnic;
2775
2776 nes_debug(NES_DBG_INIT, "\n");
2777 return sprintf(buf, "%x\n", nesvnic->nesdev->nesadapter->hw_rev);
2778}
2779
2780
2781/**
2782 * show_fw_ver
2783 */
f4e91eb4
TJ		 2784static ssize_t show_fw_ver(struct device *dev, struct device_attribute *attr,
2785 char *buf)
3c2d774c
GS		 2786{
2787 struct nes_ib_device *nesibdev =
f4e91eb4 2788 container_of(dev, struct nes_ib_device, ibdev.dev);
3c2d774c
GS		 2789 struct nes_vnic *nesvnic = nesibdev->nesvnic;
2790
2791 nes_debug(NES_DBG_INIT, "\n");
0e4562da
CT		 2792 return sprintf(buf, "%u.%u\n",
2793 (nesvnic->nesdev->nesadapter->firmware_version >> 16),
2794 (nesvnic->nesdev->nesadapter->firmware_version & 0x000000ff));
3c2d774c
GS		 2795}
2796
2797
2798/**
2799 * show_hca
2800 */
f4e91eb4
TJ		 2801static ssize_t show_hca(struct device *dev, struct device_attribute *attr,
2802 char *buf)
3c2d774c
GS		 2803{
2804 nes_debug(NES_DBG_INIT, "\n");
2805 return sprintf(buf, "NES020\n");
2806}
2807
2808
2809/**
2810 * show_board
2811 */
f4e91eb4
TJ		 2812static ssize_t show_board(struct device *dev, struct device_attribute *attr,
2813 char *buf)
3c2d774c
GS		 2814{
2815 nes_debug(NES_DBG_INIT, "\n");
2816 return sprintf(buf, "%.*s\n", 32, "NES020 Board ID");
2817}
2818
2819
f4e91eb4
TJ		 2820static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
2821static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
2822static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
2823static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
3c2d774c 2824
f4e91eb4
TJ		 2825static struct device_attribute *nes_dev_attributes[] = {
2826 &dev_attr_hw_rev,
2827 &dev_attr_fw_ver,
2828 &dev_attr_hca_type,
2829 &dev_attr_board_id
3c2d774c
GS		 2830};
2831
2832
2833/**
2834 * nes_query_qp
2835 */
2836static int nes_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2837 int attr_mask, struct ib_qp_init_attr *init_attr)
2838{
2839 struct nes_qp *nesqp = to_nesqp(ibqp);
2840
2841 nes_debug(NES_DBG_QP, "\n");
2842
2843 attr->qp_access_flags = 0;
2844 attr->cap.max_send_wr = nesqp->hwqp.sq_size;
2845 attr->cap.max_recv_wr = nesqp->hwqp.rq_size;
2846 attr->cap.max_recv_sge = 1;
eadde3a1
CT		 2847 if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA)
2848 attr->cap.max_inline_data = 0;
2849 else
2850 attr->cap.max_inline_data = 64;
3c2d774c
GS		 2851
2852 init_attr->event_handler = nesqp->ibqp.event_handler;
2853 init_attr->qp_context = nesqp->ibqp.qp_context;
2854 init_attr->send_cq = nesqp->ibqp.send_cq;
2855 init_attr->recv_cq = nesqp->ibqp.recv_cq;
4127c365 2856 init_attr->srq = nesqp->ibqp.srq;
3c2d774c
GS		 2857 init_attr->cap = attr->cap;
2858
2859 return 0;
2860}
2861
2862
2863/**
2864 * nes_hw_modify_qp
2865 */
2866int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp,
8b1c9dc4 2867 u32 next_iwarp_state, u32 termlen, u32 wait_completion)
3c2d774c
GS		 2868{
2869 struct nes_hw_cqp_wqe *cqp_wqe;
2870 /* struct iw_cm_id *cm_id = nesqp->cm_id; */
2871 /* struct iw_cm_event cm_event; */
2872 struct nes_cqp_request *cqp_request;
3c2d774c
GS		 2873 int ret;
2874 u16 major_code;
2875
2876 nes_debug(NES_DBG_MOD_QP, "QP%u, refcount=%d\n",
2877 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));
2878
2879 cqp_request = nes_get_cqp_request(nesdev);
2880 if (cqp_request == NULL) {
2881 nes_debug(NES_DBG_MOD_QP, "Failed to get a cqp_request.\n");
2882 return -ENOMEM;
2883 }
2884 if (wait_completion) {
2885 cqp_request->waiting = 1;
2886 } else {
2887 cqp_request->waiting = 0;
2888 }
2889 cqp_wqe = &cqp_request->cqp_wqe;
2890
2891 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2892 NES_CQP_MODIFY_QP | NES_CQP_QP_TYPE_IWARP | next_iwarp_state);
2893 nes_debug(NES_DBG_MOD_QP, "using next_iwarp_state=%08x, wqe_words=%08x\n",
2894 next_iwarp_state, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]));
2895 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2896 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);
2897 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, (u64)nesqp->nesqp_context_pbase);
2898
8b1c9dc4
DW		 2899 /* If sending a terminate message, fill in the length (in words) */
2900 if (((next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK) == NES_CQP_QP_IWARP_STATE_TERMINATE) &&
2901 !(next_iwarp_state & NES_CQP_QP_TERM_DONT_SEND_TERM_MSG)) {
2902 termlen = ((termlen + 3) >> 2) << NES_CQP_OP_TERMLEN_SHIFT;
2903 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_NEW_MSS_IDX, termlen);
2904 }
2905
3c2d774c 2906 atomic_set(&cqp_request->refcount, 2);
8294f297 2907 nes_post_cqp_request(nesdev, cqp_request);
3c2d774c
GS		 2908
2909 /* Wait for CQP */
2910 if (wait_completion) {
2911 /* nes_debug(NES_DBG_MOD_QP, "Waiting for modify iWARP QP%u to complete.\n",
2912 nesqp->hwqp.qp_id); */
2913 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
2914 NES_EVENT_TIMEOUT);
2915 nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u completed, wait_event_timeout ret=%u, "
2916 "CQP Major:Minor codes = 0x%04X:0x%04X.\n",
2917 nesqp->hwqp.qp_id, ret, cqp_request->major_code, cqp_request->minor_code);
2918 major_code = cqp_request->major_code;
2919 if (major_code) {
2920 nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u failed"
2921 "CQP Major:Minor codes = 0x%04X:0x%04X, intended next state = 0x%08X.\n",
2922 nesqp->hwqp.qp_id, cqp_request->major_code,
2923 cqp_request->minor_code, next_iwarp_state);
2924 }
1ff66e8c
RD		 2925
2926 nes_put_cqp_request(nesdev, cqp_request);
2927
3c2d774c
GS		 2928 if (!ret)
2929 return -ETIME;
2930 else if (major_code)
2931 return -EIO;
2932 else
2933 return 0;
2934 } else {
2935 return 0;
2936 }
2937}
2938
2939
2940/**
2941 * nes_modify_qp
2942 */
2943int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2944 int attr_mask, struct ib_udata *udata)
2945{
2946 struct nes_qp *nesqp = to_nesqp(ibqp);
2947 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device);
2948 struct nes_device *nesdev = nesvnic->nesdev;
2949 /* u32 cqp_head; */
2950 /* u32 counter; */
2951 u32 next_iwarp_state = 0;
2952 int err;
2953 unsigned long qplockflags;
2954 int ret;
2955 u16 original_last_aeq;
2956 u8 issue_modify_qp = 0;
3c2d774c
GS		 2957 u8 dont_wait = 0;
2958
2959 nes_debug(NES_DBG_MOD_QP, "QP%u: QP State=%u, cur QP State=%u,"
2960 " iwarp_state=0x%X, refcount=%d\n",
2961 nesqp->hwqp.qp_id, attr->qp_state, nesqp->ibqp_state,
2962 nesqp->iwarp_state, atomic_read(&nesqp->refcount));
2963
3c2d774c
GS		 2964 spin_lock_irqsave(&nesqp->lock, qplockflags);
2965
2966 nes_debug(NES_DBG_MOD_QP, "QP%u: hw_iwarp_state=0x%X, hw_tcp_state=0x%X,"
2967 " QP Access Flags=0x%X, attr_mask = 0x%0x\n",
2968 nesqp->hwqp.qp_id, nesqp->hw_iwarp_state,
2969 nesqp->hw_tcp_state, attr->qp_access_flags, attr_mask);
2970
2971 if (attr_mask & IB_QP_STATE) {
2972 switch (attr->qp_state) {
2973 case IB_QPS_INIT:
2974 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = init\n",
2975 nesqp->hwqp.qp_id);
2976 if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_IDLE) {
2977 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3c2d774c
GS		 2978 return -EINVAL;
2979 }
2980 next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE;
2981 issue_modify_qp = 1;
2982 break;
2983 case IB_QPS_RTR:
2984 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rtr\n",
2985 nesqp->hwqp.qp_id);
2986 if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_IDLE) {
2987 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3c2d774c
GS		 2988 return -EINVAL;
2989 }
2990 next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE;
2991 issue_modify_qp = 1;
2992 break;
2993 case IB_QPS_RTS:
2994 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rts\n",
2995 nesqp->hwqp.qp_id);
2996 if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_RTS) {
2997 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3c2d774c
GS		 2998 return -EINVAL;
2999 }
3000 if (nesqp->cm_id == NULL) {
3001 nes_debug(NES_DBG_MOD_QP, "QP%u: Failing attempt to move QP to RTS without a CM_ID. \n",
3002 nesqp->hwqp.qp_id );
3003 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3c2d774c
GS		 3004 return -EINVAL;
3005 }
3006 next_iwarp_state = NES_CQP_QP_IWARP_STATE_RTS;
3007 if (nesqp->iwarp_state != NES_CQP_QP_IWARP_STATE_RTS)
3008 next_iwarp_state |= NES_CQP_QP_CONTEXT_VALID |
3009 NES_CQP_QP_ARP_VALID | NES_CQP_QP_ORD_VALID;
3010 issue_modify_qp = 1;
3011 nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_ESTABLISHED;
3012 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_RTS;
3013 nesqp->hte_added = 1;
3014 break;
3015 case IB_QPS_SQD:
3016 issue_modify_qp = 1;
3017 nes_debug(NES_DBG_MOD_QP, "QP%u: new state=closing. SQ head=%u, SQ tail=%u\n",
3018 nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, nesqp->hwqp.sq_tail);
3019 if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_CLOSING) {
3020 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3c2d774c
GS		 3021 return 0;
3022 } else {
3023 if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_CLOSING) {
3024 nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing"
3025 " ignored due to current iWARP state\n",
3026 nesqp->hwqp.qp_id);
3027 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3c2d774c
GS		 3028 return -EINVAL;
3029 }
3030 if (nesqp->hw_iwarp_state != NES_AEQE_IWARP_STATE_RTS) {
3031 nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing"
3032 " already done based on hw state.\n",
3033 nesqp->hwqp.qp_id);
3034 issue_modify_qp = 0;
3c2d774c
GS		 3035 }
3036 switch (nesqp->hw_iwarp_state) {
3037 case NES_AEQE_IWARP_STATE_CLOSING:
3038 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
d5fb476a 3039 break;
3c2d774c
GS		 3040 case NES_AEQE_IWARP_STATE_TERMINATE:
3041 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
3042 break;
3043 case NES_AEQE_IWARP_STATE_ERROR:
3044 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3045 break;
3046 default:
3047 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3c2d774c
GS		 3048 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3049 break;
3050 }
3051 }
3052 break;
3053 case IB_QPS_SQE:
3054 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = terminate\n",
3055 nesqp->hwqp.qp_id);
3056 if (nesqp->iwarp_state>=(u32)NES_CQP_QP_IWARP_STATE_TERMINATE) {
3057 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3c2d774c
GS		 3058 return -EINVAL;
3059 }
3060 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
3061 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
3062 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
3063 issue_modify_qp = 1;
3c2d774c
GS		 3064 break;
3065 case IB_QPS_ERR:
3066 case IB_QPS_RESET:
3067 if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_ERROR) {
3068 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3c2d774c
GS		 3069 return -EINVAL;
3070 }
3071 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = error\n",
3072 nesqp->hwqp.qp_id);
8b1c9dc4
DW		 3073 if (nesqp->term_flags)
3074 del_timer(&nesqp->terminate_timer);
3075
3c2d774c
GS		 3076 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3077 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
3078 if (nesqp->hte_added) {
3079 nes_debug(NES_DBG_MOD_QP, "set CQP_QP_DEL_HTE\n");
3080 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3081 nesqp->hte_added = 0;
3082 }
3083 if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) &&
cd6860eb 3084 (nesdev->iw_status) &&
3c2d774c
GS		 3085 (nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) {
3086 next_iwarp_state |= NES_CQP_QP_RESET;
3c2d774c
GS		 3087 } else {
3088 nes_debug(NES_DBG_MOD_QP, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n",
3089 nesqp->hwqp.qp_id, nesqp->hw_tcp_state);
3090 dont_wait = 1;
3091 }
3092 issue_modify_qp = 1;
3093 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
3094 break;
3095 default:
3096 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3c2d774c
GS		 3097 return -EINVAL;
3098 break;
3099 }
3100
3101 nesqp->ibqp_state = attr->qp_state;
a67b078c
TN		 3102 nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK;
3103 nes_debug(NES_DBG_MOD_QP, "Change nesqp->iwarp_state=%08x\n",
3104 nesqp->iwarp_state);
3c2d774c
GS		 3105 }
3106
3107 if (attr_mask & IB_QP_ACCESS_FLAGS) {
3108 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) {
3109 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN |
3110 NES_QPCONTEXT_MISC_RDMA_READ_EN);
3111 issue_modify_qp = 1;
3112 }
3113 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) {
3114 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN);
3115 issue_modify_qp = 1;
3116 }
3117 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) {
3118 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN);
3119 issue_modify_qp = 1;
3120 }
3121 if (attr->qp_access_flags & IB_ACCESS_MW_BIND) {
3122 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN);
3123 issue_modify_qp = 1;
3124 }
3125
3126 if (nesqp->user_mode) {
3127 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN |
3128 NES_QPCONTEXT_MISC_RDMA_READ_EN);
3129 issue_modify_qp = 1;
3130 }
3131 }
3132
3133 original_last_aeq = nesqp->last_aeq;
3134 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3135
3136 nes_debug(NES_DBG_MOD_QP, "issue_modify_qp=%u\n", issue_modify_qp);
3137
3138 ret = 0;
3139
3140
3141 if (issue_modify_qp) {
3142 nes_debug(NES_DBG_MOD_QP, "call nes_hw_modify_qp\n");
8b1c9dc4 3143 ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 1);
3c2d774c
GS		 3144 if (ret)
3145 nes_debug(NES_DBG_MOD_QP, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)"
3146 " failed for QP%u.\n",
3147 next_iwarp_state, nesqp->hwqp.qp_id);
3148
3149 }
3150
3151 if ((issue_modify_qp) && (nesqp->ibqp_state > IB_QPS_RTS)) {
3152 nes_debug(NES_DBG_MOD_QP, "QP%u Issued ModifyQP refcount (%d),"
3153 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3154 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3155 original_last_aeq, nesqp->last_aeq);
349850f0 3156 if (!ret || original_last_aeq != NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3c2d774c
GS		 3157 if (dont_wait) {
3158 if (nesqp->cm_id && nesqp->hw_tcp_state != 0) {
3159 nes_debug(NES_DBG_MOD_QP, "QP%u Queuing fake disconnect for QP refcount (%d),"
3160 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3161 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3162 original_last_aeq, nesqp->last_aeq);
3163 /* this one is for the cm_disconnect thread */
3c2d774c
GS		 3164 spin_lock_irqsave(&nesqp->lock, qplockflags);
3165 nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3166 nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT;
3167 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3168 nes_cm_disconn(nesqp);
3169 } else {
3170 nes_debug(NES_DBG_MOD_QP, "QP%u No fake disconnect, QP refcount=%d\n",
3171 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));
3c2d774c
GS		 3172 }
3173 } else {
3174 spin_lock_irqsave(&nesqp->lock, qplockflags);
3175 if (nesqp->cm_id) {
3176 /* These two are for the timer thread */
3177 if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3c2d774c
GS		 3178 nesqp->cm_id->add_ref(nesqp->cm_id);
3179 nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d),"
3180 " need ae to finish up, original_last_aeq = 0x%04X."
3181 " last_aeq = 0x%04X, scheduling timer.\n",
3182 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3183 original_last_aeq, nesqp->last_aeq);
3184 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *) nesqp, NES_TIMER_TYPE_CLOSE, 1, 0);
3185 }
3186 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3187 } else {
3188 spin_unlock_irqrestore(&nesqp->lock, qplockflags);
3189 nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d),"
3190 " need ae to finish up, original_last_aeq = 0x%04X."
3191 " last_aeq = 0x%04X.\n",
3192 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3193 original_last_aeq, nesqp->last_aeq);
3194 }
3195 }
3196 } else {
3197 nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
3198 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3199 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3200 original_last_aeq, nesqp->last_aeq);
3c2d774c
GS		 3201 }
3202 } else {
3203 nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
3204 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3205 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3206 original_last_aeq, nesqp->last_aeq);
3c2d774c
GS		 3207 }
3208
3209 err = 0;
3210
3211 nes_debug(NES_DBG_MOD_QP, "QP%u Leaving, refcount=%d\n",
3212 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));
3213
3214 return err;
3215}
3216
3217
3218/**
3219 * nes_muticast_attach
3220 */
3221static int nes_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
3222{
3223 nes_debug(NES_DBG_INIT, "\n");
3224 return -ENOSYS;
3225}
3226
3227
3228/**
3229 * nes_multicast_detach
3230 */
3231static int nes_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
3232{
3233 nes_debug(NES_DBG_INIT, "\n");
3234 return -ENOSYS;
3235}
3236
3237
3238/**
3239 * nes_process_mad
3240 */
3241static int nes_process_mad(struct ib_device *ibdev, int mad_flags,
a97e2d86 3242 u8 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
4cd7c947
IW		 3243 const struct ib_mad_hdr *in, size_t in_mad_size,
3244 struct ib_mad_hdr *out, size_t *out_mad_size,
3245 u16 *out_mad_pkey_index)
3c2d774c
GS		 3246{
3247 nes_debug(NES_DBG_INIT, "\n");
3248 return -ENOSYS;
3249}
3250
3251static inline void
3252fill_wqe_sg_send(struct nes_hw_qp_wqe *wqe, struct ib_send_wr *ib_wr, u32 uselkey)
3253{
3254 int sge_index;
3255 int total_payload_length = 0;
3256 for (sge_index = 0; sge_index < ib_wr->num_sge; sge_index++) {
3257 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4),
3258 ib_wr->sg_list[sge_index].addr);
3259 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_LENGTH0_IDX + (sge_index*4),
3260 ib_wr->sg_list[sge_index].length);
3261 if (uselkey)
3262 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4),
3263 (ib_wr->sg_list[sge_index].lkey));
3264 else
3265 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), 0);
3266
3267 total_payload_length += ib_wr->sg_list[sge_index].length;
3268 }
3269 nes_debug(NES_DBG_IW_TX, "UC UC UC, sending total_payload_length=%u \n",
3270 total_payload_length);
3271 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
3272 total_payload_length);
3273}
3274
3275/**
3276 * nes_post_send
3277 */
3278static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
3279 struct ib_send_wr **bad_wr)
3280{
3281 u64 u64temp;
3282 unsigned long flags = 0;
3283 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device);
3284 struct nes_device *nesdev = nesvnic->nesdev;
3285 struct nes_qp *nesqp = to_nesqp(ibqp);
3286 struct nes_hw_qp_wqe *wqe;
649fe4ae 3287 int err = 0;
3c2d774c
GS		 3288 u32 qsize = nesqp->hwqp.sq_size;
3289 u32 head;
649fe4ae
CT		 3290 u32 wqe_misc = 0;
3291 u32 wqe_count = 0;
3c2d774c 3292 u32 counter;
3c2d774c 3293
e5dec394
FZ		 3294 if (nesqp->ibqp_state > IB_QPS_RTS) {
3295 err = -EINVAL;
3296 goto out;
3297 }
3c2d774c 3298
5880ff19 3299 spin_lock_irqsave(&nesqp->lock, flags);
3c2d774c
GS		 3300
3301 head = nesqp->hwqp.sq_head;
3302
3303 while (ib_wr) {
8b1c9dc4
DW		 3304 /* Check for QP error */
3305 if (nesqp->term_flags) {
3306 err = -EINVAL;
3307 break;
3308 }
3309
3c2d774c
GS		 3310 /* Check for SQ overflow */
3311 if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) {
831d06cf 3312 err = -ENOMEM;
3c2d774c
GS		 3313 break;
3314 }
3315
3316 wqe = &nesqp->hwqp.sq_vbase[head];
3317 /* nes_debug(NES_DBG_IW_TX, "processing sq wqe for QP%u at %p, head = %u.\n",
3318 nesqp->hwqp.qp_id, wqe, head); */
3319 nes_fill_init_qp_wqe(wqe, nesqp, head);
3320 u64temp = (u64)(ib_wr->wr_id);
3321 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX,
3322 u64temp);
649fe4ae
CT		 3323 switch (ib_wr->opcode) {
3324 case IB_WR_SEND:
3325 case IB_WR_SEND_WITH_INV:
3326 if (IB_WR_SEND == ib_wr->opcode) {
3327 if (ib_wr->send_flags & IB_SEND_SOLICITED)
3328 wqe_misc = NES_IWARP_SQ_OP_SENDSE;
3329 else
3330 wqe_misc = NES_IWARP_SQ_OP_SEND;
3331 } else {
3332 if (ib_wr->send_flags & IB_SEND_SOLICITED)
3333 wqe_misc = NES_IWARP_SQ_OP_SENDSEINV;
3334 else
3335 wqe_misc = NES_IWARP_SQ_OP_SENDINV;
3c2d774c 3336
649fe4ae
CT		 3337 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX,
3338 ib_wr->ex.invalidate_rkey);
3339 }
3c2d774c 3340
649fe4ae
CT		 3341 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
3342 err = -EINVAL;
3343 break;
3c2d774c
GS		 3344 }
3345
649fe4ae
CT		 3346 if (ib_wr->send_flags & IB_SEND_FENCE)
3347 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
3348
3349 if ((ib_wr->send_flags & IB_SEND_INLINE) &&
3350 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
3351 (ib_wr->sg_list[0].length <= 64)) {
3352 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
3353 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
3354 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
3355 ib_wr->sg_list[0].length);
3356 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
3357 } else {
3358 fill_wqe_sg_send(wqe, ib_wr, 1);
3359 }
3360
3361 break;
3362 case IB_WR_RDMA_WRITE:
3363 wqe_misc = NES_IWARP_SQ_OP_RDMAW;
3364 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
3365 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n",
3366 ib_wr->num_sge, nesdev->nesadapter->max_sge);
3367 err = -EINVAL;
3368 break;
3369 }
3370
3371 if (ib_wr->send_flags & IB_SEND_FENCE)
3372 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
3373
3374 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
3375 ib_wr->wr.rdma.rkey);
3376 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
3377 ib_wr->wr.rdma.remote_addr);
3378
3379 if ((ib_wr->send_flags & IB_SEND_INLINE) &&
3380 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
3381 (ib_wr->sg_list[0].length <= 64)) {
3382 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
3383 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
3384 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
3385 ib_wr->sg_list[0].length);
3386 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
3387 } else {
3388 fill_wqe_sg_send(wqe, ib_wr, 1);
3389 }
3390
3391 wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] =
3392 wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX];
3393 break;
3394 case IB_WR_RDMA_READ:
3395 case IB_WR_RDMA_READ_WITH_INV:
3396 /* iWARP only supports 1 sge for RDMA reads */
3397 if (ib_wr->num_sge > 1) {
3398 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n",
3399 ib_wr->num_sge);
3400 err = -EINVAL;
3401 break;
3402 }
3403 if (ib_wr->opcode == IB_WR_RDMA_READ) {
3404 wqe_misc = NES_IWARP_SQ_OP_RDMAR;
3405 } else {
3406 wqe_misc = NES_IWARP_SQ_OP_RDMAR_LOCINV;
3407 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX,
3408 ib_wr->ex.invalidate_rkey);
3409 }
3410
3411 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
3412 ib_wr->wr.rdma.remote_addr);
3413 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
3414 ib_wr->wr.rdma.rkey);
3415 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX,
3416 ib_wr->sg_list->length);
3417 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX,
3418 ib_wr->sg_list->addr);
3419 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX,
3420 ib_wr->sg_list->lkey);
3421 break;
3422 case IB_WR_LOCAL_INV:
3423 wqe_misc = NES_IWARP_SQ_OP_LOCINV;
3424 set_wqe_32bit_value(wqe->wqe_words,
3425 NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX,
3426 ib_wr->ex.invalidate_rkey);
3427 break;
e293a26f
CT		 3428 case IB_WR_FAST_REG_MR:
3429 {
3430 int i;
3431 int flags = ib_wr->wr.fast_reg.access_flags;
3432 struct nes_ib_fast_reg_page_list *pnesfrpl =
3433 container_of(ib_wr->wr.fast_reg.page_list,
3434 struct nes_ib_fast_reg_page_list,
3435 ibfrpl);
3436 u64 *src_page_list = pnesfrpl->ibfrpl.page_list;
3437 u64 *dst_page_list = pnesfrpl->nes_wqe_pbl.kva;
3438
3439 if (ib_wr->wr.fast_reg.page_list_len >
3440 (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) {
3441 nes_debug(NES_DBG_IW_TX, "SQ_FMR: bad page_list_len\n");
3442 err = -EINVAL;
3443 break;
3444 }
3445 wqe_misc = NES_IWARP_SQ_OP_FAST_REG;
3446 set_wqe_64bit_value(wqe->wqe_words,
3447 NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX,
3448 ib_wr->wr.fast_reg.iova_start);
3449 set_wqe_32bit_value(wqe->wqe_words,
3450 NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX,
3451 ib_wr->wr.fast_reg.length);
94f622bd
TN		 3452 set_wqe_32bit_value(wqe->wqe_words,
3453 NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX, 0);
e293a26f
CT		 3454 set_wqe_32bit_value(wqe->wqe_words,
3455 NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX,
3456 ib_wr->wr.fast_reg.rkey);
3457 /* Set page size: */
3458 if (ib_wr->wr.fast_reg.page_shift == 12) {
3459 wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K;
3460 } else if (ib_wr->wr.fast_reg.page_shift == 21) {
3461 wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M;
3462 } else {
3463 nes_debug(NES_DBG_IW_TX, "Invalid page shift,"
3464 " ib_wr=%u, max=1\n", ib_wr->num_sge);
3465 err = -EINVAL;
3466 break;
3467 }
3468 /* Set access_flags */
3469 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ;
3470 if (flags & IB_ACCESS_LOCAL_WRITE)
3471 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE;
3472
3473 if (flags & IB_ACCESS_REMOTE_WRITE)
3474 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE;
3475
3476 if (flags & IB_ACCESS_REMOTE_READ)
3477 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ;
3478
3479 if (flags & IB_ACCESS_MW_BIND)
3480 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND;
3481
3482 /* Fill in PBL info: */
3483 if (ib_wr->wr.fast_reg.page_list_len >
3484 pnesfrpl->ibfrpl.max_page_list_len) {
3485 nes_debug(NES_DBG_IW_TX, "Invalid page list length,"
3486 " ib_wr=%p, value=%u, max=%u\n",
3487 ib_wr, ib_wr->wr.fast_reg.page_list_len,
3488 pnesfrpl->ibfrpl.max_page_list_len);
3489 err = -EINVAL;
3490 break;
3491 }
3492
3493 set_wqe_64bit_value(wqe->wqe_words,
3494 NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX,
3495 pnesfrpl->nes_wqe_pbl.paddr);
3496
3497 set_wqe_32bit_value(wqe->wqe_words,
3498 NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX,
3499 ib_wr->wr.fast_reg.page_list_len * 8);
3500
3501 for (i = 0; i < ib_wr->wr.fast_reg.page_list_len; i++)
3502 dst_page_list[i] = cpu_to_le64(src_page_list[i]);
3503
183ae74b
RD		 3504 nes_debug(NES_DBG_IW_TX, "SQ_FMR: iova_start: %llx, "
3505 "length: %d, rkey: %0x, pgl_paddr: %llx, "
e293a26f 3506 "page_list_len: %u, wqe_misc: %x\n",
183ae74b 3507 (unsigned long long) ib_wr->wr.fast_reg.iova_start,
e293a26f
CT		 3508 ib_wr->wr.fast_reg.length,
3509 ib_wr->wr.fast_reg.rkey,
183ae74b 3510 (unsigned long long) pnesfrpl->nes_wqe_pbl.paddr,
e293a26f
CT		 3511 ib_wr->wr.fast_reg.page_list_len,
3512 wqe_misc);
3513 break;
3514 }
649fe4ae
CT		 3515 default:
3516 /* error */
3517 err = -EINVAL;
3518 break;
3519 }
3520
e5dec394
FZ		 3521 if (err)
3522 break;
3523
d14152da 3524 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || nesqp->sig_all)
3c2d774c 3525 wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL;
d14152da 3526
3c2d774c
GS		 3527 wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(wqe_misc);
3528
3529 ib_wr = ib_wr->next;
3530 head++;
3531 wqe_count++;
3532 if (head >= qsize)
3533 head = 0;
3534
3535 }
3536
3537 nesqp->hwqp.sq_head = head;
3538 barrier();
3539 while (wqe_count) {
3540 counter = min(wqe_count, ((u32)255));
3541 wqe_count -= counter;
3542 nes_write32(nesdev->regs + NES_WQE_ALLOC,
3543 (counter << 24) | 0x00800000 | nesqp->hwqp.qp_id);
3544 }
3545
5880ff19 3546 spin_unlock_irqrestore(&nesqp->lock, flags);
3c2d774c 3547
e5dec394 3548out:
3c2d774c
GS		 3549 if (err)
3550 *bad_wr = ib_wr;
3551 return err;
3552}
3553
3554
3555/**
3556 * nes_post_recv
3557 */
3558static int nes_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
3559 struct ib_recv_wr **bad_wr)
3560{
3561 u64 u64temp;
3562 unsigned long flags = 0;
3563 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device);
3564 struct nes_device *nesdev = nesvnic->nesdev;
3565 struct nes_qp *nesqp = to_nesqp(ibqp);
3566 struct nes_hw_qp_wqe *wqe;
3567 int err = 0;
3568 int sge_index;
3569 u32 qsize = nesqp->hwqp.rq_size;
3570 u32 head;
3571 u32 wqe_count = 0;
3572 u32 counter;
3573 u32 total_payload_length;
3574
4293fdc1
FZ		 3575 if (nesqp->ibqp_state > IB_QPS_RTS) {
3576 err = -EINVAL;
3577 goto out;
3578 }
3c2d774c 3579
5880ff19 3580 spin_lock_irqsave(&nesqp->lock, flags);
3c2d774c
GS		 3581
3582 head = nesqp->hwqp.rq_head;
3583
3584 while (ib_wr) {
8b1c9dc4
DW		 3585 /* Check for QP error */
3586 if (nesqp->term_flags) {
3587 err = -EINVAL;
3588 break;
3589 }
3590
3c2d774c
GS		 3591 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
3592 err = -EINVAL;
3593 break;
3594 }
3595 /* Check for RQ overflow */
3596 if (((head + (2 * qsize) - nesqp->hwqp.rq_tail) % qsize) == (qsize - 1)) {
831d06cf 3597 err = -ENOMEM;
3c2d774c
GS		 3598 break;
3599 }
3600
3601 nes_debug(NES_DBG_IW_RX, "ibwr sge count = %u.\n", ib_wr->num_sge);
3602 wqe = &nesqp->hwqp.rq_vbase[head];
3603
3604 /* nes_debug(NES_DBG_IW_RX, "QP%u:processing rq wqe at %p, head = %u.\n",
3605 nesqp->hwqp.qp_id, wqe, head); */
3606 nes_fill_init_qp_wqe(wqe, nesqp, head);
3607 u64temp = (u64)(ib_wr->wr_id);
3608 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX,
3609 u64temp);
3610 total_payload_length = 0;
3611 for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
3612 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4),
3613 ib_wr->sg_list[sge_index].addr);
3614 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4),
3615 ib_wr->sg_list[sge_index].length);
3616 set_wqe_32bit_value(wqe->wqe_words,NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4),
3617 ib_wr->sg_list[sge_index].lkey);
3618
3619 total_payload_length += ib_wr->sg_list[sge_index].length;
3620 }
3621 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX,
3622 total_payload_length);
3623
3624 ib_wr = ib_wr->next;
3625 head++;
3626 wqe_count++;
3627 if (head >= qsize)
3628 head = 0;
3629 }
3630
3631 nesqp->hwqp.rq_head = head;
3632 barrier();
3633 while (wqe_count) {
3634 counter = min(wqe_count, ((u32)255));
3635 wqe_count -= counter;
3636 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter<<24) | nesqp->hwqp.qp_id);
3637 }
3638
5880ff19 3639 spin_unlock_irqrestore(&nesqp->lock, flags);
3c2d774c 3640
4293fdc1 3641out:
3c2d774c
GS		 3642 if (err)
3643 *bad_wr = ib_wr;
3644 return err;
3645}
3646
3647
3648/**
3649 * nes_poll_cq
3650 */
3651static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
3652{
3653 u64 u64temp;
3654 u64 wrid;
3c2d774c
GS		 3655 unsigned long flags = 0;
3656 struct nes_vnic *nesvnic = to_nesvnic(ibcq->device);
3657 struct nes_device *nesdev = nesvnic->nesdev;
3658 struct nes_cq *nescq = to_nescq(ibcq);
3659 struct nes_qp *nesqp;
3660 struct nes_hw_cqe cqe;
3661 u32 head;
6eed5e7c 3662 u32 wq_tail = 0;
3c2d774c
GS		 3663 u32 cq_size;
3664 u32 cqe_count = 0;
3665 u32 wqe_index;
3666 u32 u32temp;
6eed5e7c
DW		 3667 u32 move_cq_head = 1;
3668 u32 err_code;
3c2d774c
GS		 3669
3670 nes_debug(NES_DBG_CQ, "\n");
3671
5880ff19 3672 spin_lock_irqsave(&nescq->lock, flags);
3c2d774c
GS		 3673
3674 head = nescq->hw_cq.cq_head;
3675 cq_size = nescq->hw_cq.cq_size;
3676
3677 while (cqe_count < num_entries) {
5ee21fe0
DW		 3678 if ((le32_to_cpu(nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) &
3679 NES_CQE_VALID) == 0)
3680 break;
3681
3682 /*
3683 * Make sure we read CQ entry contents *after*
3684 * we've checked the valid bit.
3685 */
3686 rmb();
3687
3688 cqe = nescq->hw_cq.cq_vbase[head];
5ee21fe0
DW		 3689 u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
3690 wqe_index = u32temp & (nesdev->nesadapter->max_qp_wr - 1);
3691 u32temp &= ~(NES_SW_CONTEXT_ALIGN-1);
3692 /* parse CQE, get completion context from WQE (either rq or sq) */
3693 u64temp = (((u64)(le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) |
3694 ((u64)u32temp);
3695
3696 if (u64temp) {
3697 nesqp = (struct nes_qp *)(unsigned long)u64temp;
3c2d774c
GS		 3698 memset(entry, 0, sizeof *entry);
3699 if (cqe.cqe_words[NES_CQE_ERROR_CODE_IDX] == 0) {
3700 entry->status = IB_WC_SUCCESS;
3701 } else {
320cdfd2
DW		 3702 err_code = le32_to_cpu(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]);
3703 if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) {
3704 entry->status = err_code & 0x0000ffff;
3705
3706 /* The rest of the cqe's will be marked as flushed */
3707 nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX] =
3708 cpu_to_le32((NES_IWARP_CQE_MAJOR_FLUSH << 16) |
3709 NES_IWARP_CQE_MINOR_FLUSH);
3710 } else
3711 entry->status = IB_WC_WR_FLUSH_ERR;
3c2d774c
GS		 3712 }
3713
3714 entry->qp = &nesqp->ibqp;
3715 entry->src_qp = nesqp->hwqp.qp_id;
3716
3717 if (le32_to_cpu(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) {
3718 if (nesqp->skip_lsmm) {
3719 nesqp->skip_lsmm = 0;
5ee21fe0 3720 nesqp->hwqp.sq_tail++;
3c2d774c
GS		 3721 }
3722
3723 /* Working on a SQ Completion*/
6eed5e7c 3724 wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index].
3c2d774c 3725 wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
6eed5e7c 3726 ((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index].
3c2d774c 3727 wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX])));
6eed5e7c 3728 entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].
3c2d774c
GS		 3729 wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]);
3730
6eed5e7c 3731 switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].
3c2d774c
GS		 3732 wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) {
3733 case NES_IWARP_SQ_OP_RDMAW:
3734 nes_debug(NES_DBG_CQ, "Operation = RDMA WRITE.\n");
3735 entry->opcode = IB_WC_RDMA_WRITE;
3736 break;
3737 case NES_IWARP_SQ_OP_RDMAR:
3738 nes_debug(NES_DBG_CQ, "Operation = RDMA READ.\n");
3739 entry->opcode = IB_WC_RDMA_READ;
6eed5e7c 3740 entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].
3c2d774c
GS		 3741 wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]);
3742 break;
3743 case NES_IWARP_SQ_OP_SENDINV:
3744 case NES_IWARP_SQ_OP_SENDSEINV:
3745 case NES_IWARP_SQ_OP_SEND:
3746 case NES_IWARP_SQ_OP_SENDSE:
3747 nes_debug(NES_DBG_CQ, "Operation = Send.\n");
3748 entry->opcode = IB_WC_SEND;
3749 break;
649fe4ae 3750 case NES_IWARP_SQ_OP_LOCINV:
4a4b03f4 3751 entry->opcode = IB_WC_LOCAL_INV;
649fe4ae 3752 break;
e293a26f
CT		 3753 case NES_IWARP_SQ_OP_FAST_REG:
3754 entry->opcode = IB_WC_FAST_REG_MR;
3755 break;
3c2d774c 3756 }
6eed5e7c
DW		 3757
3758 nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1);
3759 if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head)) {
3760 move_cq_head = 0;
3761 wq_tail = nesqp->hwqp.sq_tail;
3762 }
3c2d774c
GS		 3763 } else {
3764 /* Working on a RQ Completion*/
3c2d774c 3765 entry->byte_len = le32_to_cpu(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]);
6eed5e7c
DW		 3766 wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX]))) |
3767 ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
3c2d774c 3768 entry->opcode = IB_WC_RECV;
6eed5e7c
DW		 3769
3770 nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1);
3771 if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head)) {
3772 move_cq_head = 0;
3773 wq_tail = nesqp->hwqp.rq_tail;
3774 }
3c2d774c 3775 }
3c2d774c 3776
5ee21fe0 3777 entry->wr_id = wrid;
3c2d774c 3778 entry++;
5ee21fe0
DW		 3779 cqe_count++;
3780 }
3781
6eed5e7c
DW		 3782 if (move_cq_head) {
3783 nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3784 if (++head >= cq_size)
3785 head = 0;
3786 nescq->polled_completions++;
5ee21fe0 3787
6eed5e7c
DW		 3788 if ((nescq->polled_completions > (cq_size / 2)) ||
3789 (nescq->polled_completions == 255)) {
3790 nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes"
5ee21fe0
DW		 3791 " are pending %u of %u.\n",
3792 nescq->hw_cq.cq_number, nescq->polled_completions, cq_size);
6eed5e7c 3793 nes_write32(nesdev->regs+NES_CQE_ALLOC,
5ee21fe0 3794 nescq->hw_cq.cq_number | (nescq->polled_completions << 16));
6eed5e7c
DW		 3795 nescq->polled_completions = 0;
3796 }
3797 } else {
3798 /* Update the wqe index and set status to flush */
3799 wqe_index = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
3800 wqe_index = (wqe_index & (~(nesdev->nesadapter->max_qp_wr - 1))) | wq_tail;
3801 nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] =
3802 cpu_to_le32(wqe_index);
3803 move_cq_head = 1; /* ready for next pass */
5ee21fe0 3804 }
3c2d774c
GS		 3805 }
3806
3807 if (nescq->polled_completions) {
3808 nes_write32(nesdev->regs+NES_CQE_ALLOC,
3809 nescq->hw_cq.cq_number | (nescq->polled_completions << 16));
3810 nescq->polled_completions = 0;
3811 }
3812
3813 nescq->hw_cq.cq_head = head;
3814 nes_debug(NES_DBG_CQ, "Reporting %u completions for CQ%u.\n",
3815 cqe_count, nescq->hw_cq.cq_number);
3816
5880ff19 3817 spin_unlock_irqrestore(&nescq->lock, flags);
3c2d774c
GS		 3818
3819 return cqe_count;
3820}
3821
3822
3823/**
3824 * nes_req_notify_cq
3825 */
3826static int nes_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags)
3827 {
3828 struct nes_vnic *nesvnic = to_nesvnic(ibcq->device);
3829 struct nes_device *nesdev = nesvnic->nesdev;
3830 struct nes_cq *nescq = to_nescq(ibcq);
3831 u32 cq_arm;
3832
3833 nes_debug(NES_DBG_CQ, "Requesting notification for CQ%u.\n",
3834 nescq->hw_cq.cq_number);
3835
3836 cq_arm = nescq->hw_cq.cq_number;
3837 if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_NEXT_COMP)
3838 cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT;
3839 else if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
3840 cq_arm |= NES_CQE_ALLOC_NOTIFY_SE;
3841 else
3842 return -EINVAL;
3843
3844 nes_write32(nesdev->regs+NES_CQE_ALLOC, cq_arm);
3845 nes_read32(nesdev->regs+NES_CQE_ALLOC);
3846
3847 return 0;
3848}
3849
7738613e
IW		 3850static int nes_port_immutable(struct ib_device *ibdev, u8 port_num,
3851 struct ib_port_immutable *immutable)
3852{
3853 struct ib_port_attr attr;
3854 int err;
3855
3856 err = nes_query_port(ibdev, port_num, &attr);
3857 if (err)
3858 return err;
3859
3860 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3861 immutable->gid_tbl_len = attr.gid_tbl_len;
f9b22e35 3862 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
7738613e
IW		 3863
3864 return 0;
3865}
3c2d774c
GS		 3866
3867/**
3868 * nes_init_ofa_device
3869 */
3870struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev)
3871{
3872 struct nes_ib_device *nesibdev;
3873 struct nes_vnic *nesvnic = netdev_priv(netdev);
3874 struct nes_device *nesdev = nesvnic->nesdev;
3875
3876 nesibdev = (struct nes_ib_device *)ib_alloc_device(sizeof(struct nes_ib_device));
3877 if (nesibdev == NULL) {
3878 return NULL;
3879 }
3880 strlcpy(nesibdev->ibdev.name, "nes%d", IB_DEVICE_NAME_MAX);
3881 nesibdev->ibdev.owner = THIS_MODULE;
3882
3883 nesibdev->ibdev.node_type = RDMA_NODE_RNIC;
3884 memset(&nesibdev->ibdev.node_guid, 0, sizeof(nesibdev->ibdev.node_guid));
3885 memcpy(&nesibdev->ibdev.node_guid, netdev->dev_addr, 6);
3886
3887 nesibdev->ibdev.uverbs_cmd_mask =
3888 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
3889 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
3890 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
3891 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
3892 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
3893 (1ull << IB_USER_VERBS_CMD_REG_MR) |
3894 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
3895 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
3896 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
3897 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
3898 (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
3899 (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
3900 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
3901 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
3902 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
3903 (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
3904 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
3905 (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
3906 (1ull << IB_USER_VERBS_CMD_BIND_MW) |
3907 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) |
3908 (1ull << IB_USER_VERBS_CMD_POST_RECV) |
3909 (1ull << IB_USER_VERBS_CMD_POST_SEND);
3910
3911 nesibdev->ibdev.phys_port_cnt = 1;
3912 nesibdev->ibdev.num_comp_vectors = 1;
3913 nesibdev->ibdev.dma_device = &nesdev->pcidev->dev;
f4e91eb4 3914 nesibdev->ibdev.dev.parent = &nesdev->pcidev->dev;
3c2d774c
GS		 3915 nesibdev->ibdev.query_device = nes_query_device;
3916 nesibdev->ibdev.query_port = nes_query_port;
3c2d774c
GS		 3917 nesibdev->ibdev.query_pkey = nes_query_pkey;
3918 nesibdev->ibdev.query_gid = nes_query_gid;
3919 nesibdev->ibdev.alloc_ucontext = nes_alloc_ucontext;
3920 nesibdev->ibdev.dealloc_ucontext = nes_dealloc_ucontext;
3921 nesibdev->ibdev.mmap = nes_mmap;
3922 nesibdev->ibdev.alloc_pd = nes_alloc_pd;
3923 nesibdev->ibdev.dealloc_pd = nes_dealloc_pd;
3924 nesibdev->ibdev.create_ah = nes_create_ah;
3925 nesibdev->ibdev.destroy_ah = nes_destroy_ah;
3926 nesibdev->ibdev.create_qp = nes_create_qp;
3927 nesibdev->ibdev.modify_qp = nes_modify_qp;
3928 nesibdev->ibdev.query_qp = nes_query_qp;
3929 nesibdev->ibdev.destroy_qp = nes_destroy_qp;
3930 nesibdev->ibdev.create_cq = nes_create_cq;
3931 nesibdev->ibdev.destroy_cq = nes_destroy_cq;
3932 nesibdev->ibdev.poll_cq = nes_poll_cq;
3933 nesibdev->ibdev.get_dma_mr = nes_get_dma_mr;
3934 nesibdev->ibdev.reg_phys_mr = nes_reg_phys_mr;
3935 nesibdev->ibdev.reg_user_mr = nes_reg_user_mr;
3936 nesibdev->ibdev.dereg_mr = nes_dereg_mr;
3937 nesibdev->ibdev.alloc_mw = nes_alloc_mw;
3938 nesibdev->ibdev.dealloc_mw = nes_dealloc_mw;
3939 nesibdev->ibdev.bind_mw = nes_bind_mw;
3940
e02e4d55 3941 nesibdev->ibdev.alloc_mr = nes_alloc_mr;
e293a26f
CT		 3942 nesibdev->ibdev.alloc_fast_reg_page_list = nes_alloc_fast_reg_page_list;
3943 nesibdev->ibdev.free_fast_reg_page_list = nes_free_fast_reg_page_list;
3c2d774c
GS		 3944
3945 nesibdev->ibdev.attach_mcast = nes_multicast_attach;
3946 nesibdev->ibdev.detach_mcast = nes_multicast_detach;
3947 nesibdev->ibdev.process_mad = nes_process_mad;
3948
3949 nesibdev->ibdev.req_notify_cq = nes_req_notify_cq;
3950 nesibdev->ibdev.post_send = nes_post_send;
3951 nesibdev->ibdev.post_recv = nes_post_recv;
3952
3953 nesibdev->ibdev.iwcm = kzalloc(sizeof(*nesibdev->ibdev.iwcm), GFP_KERNEL);
3954 if (nesibdev->ibdev.iwcm == NULL) {
3955 ib_dealloc_device(&nesibdev->ibdev);
3956 return NULL;
3957 }
3958 nesibdev->ibdev.iwcm->add_ref = nes_add_ref;
3959 nesibdev->ibdev.iwcm->rem_ref = nes_rem_ref;
3960 nesibdev->ibdev.iwcm->get_qp = nes_get_qp;
3961 nesibdev->ibdev.iwcm->connect = nes_connect;
3962 nesibdev->ibdev.iwcm->accept = nes_accept;
3963 nesibdev->ibdev.iwcm->reject = nes_reject;
3964 nesibdev->ibdev.iwcm->create_listen = nes_create_listen;
3965 nesibdev->ibdev.iwcm->destroy_listen = nes_destroy_listen;
7738613e 3966 nesibdev->ibdev.get_port_immutable = nes_port_immutable;
3c2d774c
GS		 3967
3968 return nesibdev;
3969}
3970
ea623455
MS		 3971
3972/**
3973 * nes_handle_delayed_event
3974 */
3975static void nes_handle_delayed_event(unsigned long data)
3976{
3977 struct nes_vnic *nesvnic = (void *) data;
3978
3979 if (nesvnic->delayed_event != nesvnic->last_dispatched_event) {
3980 struct ib_event event;
3981
3982 event.device = &nesvnic->nesibdev->ibdev;
3983 if (!event.device)
3984 goto stop_timer;
3985 event.event = nesvnic->delayed_event;
3986 event.element.port_num = nesvnic->logical_port + 1;
3987 ib_dispatch_event(&event);
3988 }
3989
3990stop_timer:
3991 nesvnic->event_timer.function = NULL;
3992}
3993
3994
cd6860eb
FL		 3995void nes_port_ibevent(struct nes_vnic *nesvnic)
3996{
3997 struct nes_ib_device *nesibdev = nesvnic->nesibdev;
3998 struct nes_device *nesdev = nesvnic->nesdev;
3999 struct ib_event event;
4000 event.device = &nesibdev->ibdev;
4001 event.element.port_num = nesvnic->logical_port + 1;
4002 event.event = nesdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
ea623455
MS		 4003
4004 if (!nesvnic->event_timer.function) {
4005 ib_dispatch_event(&event);
4006 nesvnic->last_dispatched_event = event.event;
4007 nesvnic->event_timer.function = nes_handle_delayed_event;
4008 nesvnic->event_timer.data = (unsigned long) nesvnic;
4009 nesvnic->event_timer.expires = jiffies + NES_EVENT_DELAY;
4010 add_timer(&nesvnic->event_timer);
4011 } else {
4012 mod_timer(&nesvnic->event_timer, jiffies + NES_EVENT_DELAY);
4013 }
4014 nesvnic->delayed_event = event.event;
cd6860eb
FL		 4015}
4016
3c2d774c
GS		 4017
4018/**
4019 * nes_destroy_ofa_device
4020 */
4021void nes_destroy_ofa_device(struct nes_ib_device *nesibdev)
4022{
4023 if (nesibdev == NULL)
4024 return;
4025
4026 nes_unregister_ofa_device(nesibdev);
4027
4028 kfree(nesibdev->ibdev.iwcm);
4029 ib_dealloc_device(&nesibdev->ibdev);
4030}
4031
4032
4033/**
4034 * nes_register_ofa_device
4035 */
4036int nes_register_ofa_device(struct nes_ib_device *nesibdev)
4037{
4038 struct nes_vnic *nesvnic = nesibdev->nesvnic;
4039 struct nes_device *nesdev = nesvnic->nesdev;
4040 struct nes_adapter *nesadapter = nesdev->nesadapter;
4041 int i, ret;
4042
9a6edb60 4043 ret = ib_register_device(&nesvnic->nesibdev->ibdev, NULL);
3c2d774c
GS		 4044 if (ret) {
4045 return ret;
4046 }
4047
4048 /* Get the resources allocated to this device */
4049 nesibdev->max_cq = (nesadapter->max_cq-NES_FIRST_QPN) / nesadapter->port_count;
4050 nesibdev->max_mr = nesadapter->max_mr / nesadapter->port_count;
4051 nesibdev->max_qp = (nesadapter->max_qp-NES_FIRST_QPN) / nesadapter->port_count;
4052 nesibdev->max_pd = nesadapter->max_pd / nesadapter->port_count;
4053
f4e91eb4
TJ		 4054 for (i = 0; i < ARRAY_SIZE(nes_dev_attributes); ++i) {
4055 ret = device_create_file(&nesibdev->ibdev.dev, nes_dev_attributes[i]);
3c2d774c
GS		 4056 if (ret) {
4057 while (i > 0) {
4058 i--;
f4e91eb4
TJ		 4059 device_remove_file(&nesibdev->ibdev.dev,
4060 nes_dev_attributes[i]);
3c2d774c
GS		 4061 }
4062 ib_unregister_device(&nesibdev->ibdev);
4063 return ret;
4064 }
4065 }
4066
4067 nesvnic->of_device_registered = 1;
4068
4069 return 0;
4070}
4071
4072
4073/**
4074 * nes_unregister_ofa_device
4075 */
1a855fbf 4076static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev)
3c2d774c
GS		 4077{
4078 struct nes_vnic *nesvnic = nesibdev->nesvnic;
4079 int i;
4080
f4e91eb4
TJ		 4081 for (i = 0; i < ARRAY_SIZE(nes_dev_attributes); ++i) {
4082 device_remove_file(&nesibdev->ibdev.dev, nes_dev_attributes[i]);
3c2d774c
GS		 4083 }
4084
4085 if (nesvnic->of_device_registered) {
4086 ib_unregister_device(&nesibdev->ibdev);
4087 }
4088
4089 nesvnic->of_device_registered = 0;
4090}
Linux kernel - Deliverables
This page took 0.662107 seconds and 5 git commands to generate.