1 /*******************************************************************************
3 * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 *******************************************************************************/
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/netdevice.h>
38 #include <linux/etherdevice.h>
40 #include <linux/tcp.h>
41 #include <linux/if_vlan.h>
42 #include <net/addrconf.h>
45 #include "i40iw_register.h"
46 #include <net/netevent.h>
47 #define CLIENT_IW_INTERFACE_VERSION_MAJOR 0
48 #define CLIENT_IW_INTERFACE_VERSION_MINOR 01
49 #define CLIENT_IW_INTERFACE_VERSION_BUILD 00
51 #define DRV_VERSION_MAJOR 0
52 #define DRV_VERSION_MINOR 5
53 #define DRV_VERSION_BUILD 123
54 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
55 __stringify(DRV_VERSION_MINOR) "." __stringify(DRV_VERSION_BUILD)
58 module_param(push_mode
, int, 0644);
59 MODULE_PARM_DESC(push_mode
, "Low latency mode: 0=disabled (default), 1=enabled)");
62 module_param(debug
, int, 0644);
63 MODULE_PARM_DESC(debug
, "debug flags: 0=disabled (default), 0x7fffffff=all");
65 static int resource_profile
;
66 module_param(resource_profile
, int, 0644);
67 MODULE_PARM_DESC(resource_profile
,
68 "Resource Profile: 0=no VF RDMA support (default), 1=Weighted VF, 2=Even Distribution");
70 static int max_rdma_vfs
= 32;
71 module_param(max_rdma_vfs
, int, 0644);
72 MODULE_PARM_DESC(max_rdma_vfs
, "Maximum VF count: 0-32 32=default");
73 static int mpa_version
= 2;
74 module_param(mpa_version
, int, 0644);
75 MODULE_PARM_DESC(mpa_version
, "MPA version to be used in MPA Req/Resp 1 or 2");
77 MODULE_AUTHOR("Intel Corporation, <e1000-rdma@lists.sourceforge.net>");
78 MODULE_DESCRIPTION("Intel(R) Ethernet Connection X722 iWARP RDMA Driver");
79 MODULE_LICENSE("Dual BSD/GPL");
80 MODULE_VERSION(DRV_VERSION
);
82 static struct i40e_client i40iw_client
;
83 static char i40iw_client_name
[I40E_CLIENT_STR_LENGTH
] = "i40iw";
85 static LIST_HEAD(i40iw_handlers
);
86 static spinlock_t i40iw_handler_lock
;
88 static enum i40iw_status_code
i40iw_virtchnl_send(struct i40iw_sc_dev
*dev
,
89 u32 vf_id
, u8
*msg
, u16 len
);
91 static struct notifier_block i40iw_inetaddr_notifier
= {
92 .notifier_call
= i40iw_inetaddr_event
95 static struct notifier_block i40iw_inetaddr6_notifier
= {
96 .notifier_call
= i40iw_inet6addr_event
99 static struct notifier_block i40iw_net_notifier
= {
100 .notifier_call
= i40iw_net_event
103 static int i40iw_notifiers_registered
;
106 * i40iw_find_i40e_handler - find a handler given a client info
107 * @ldev: pointer to a client info
109 static struct i40iw_handler
*i40iw_find_i40e_handler(struct i40e_info
*ldev
)
111 struct i40iw_handler
*hdl
;
114 spin_lock_irqsave(&i40iw_handler_lock
, flags
);
115 list_for_each_entry(hdl
, &i40iw_handlers
, list
) {
116 if (hdl
->ldev
.netdev
== ldev
->netdev
) {
117 spin_unlock_irqrestore(&i40iw_handler_lock
, flags
);
121 spin_unlock_irqrestore(&i40iw_handler_lock
, flags
);
126 * i40iw_find_netdev - find a handler given a netdev
127 * @netdev: pointer to net_device
129 struct i40iw_handler
*i40iw_find_netdev(struct net_device
*netdev
)
131 struct i40iw_handler
*hdl
;
134 spin_lock_irqsave(&i40iw_handler_lock
, flags
);
135 list_for_each_entry(hdl
, &i40iw_handlers
, list
) {
136 if (hdl
->ldev
.netdev
== netdev
) {
137 spin_unlock_irqrestore(&i40iw_handler_lock
, flags
);
141 spin_unlock_irqrestore(&i40iw_handler_lock
, flags
);
146 * i40iw_add_handler - add a handler to the list
147 * @hdl: handler to be added to the handler list
149 static void i40iw_add_handler(struct i40iw_handler
*hdl
)
153 spin_lock_irqsave(&i40iw_handler_lock
, flags
);
154 list_add(&hdl
->list
, &i40iw_handlers
);
155 spin_unlock_irqrestore(&i40iw_handler_lock
, flags
);
159 * i40iw_del_handler - delete a handler from the list
160 * @hdl: handler to be deleted from the handler list
162 static int i40iw_del_handler(struct i40iw_handler
*hdl
)
166 spin_lock_irqsave(&i40iw_handler_lock
, flags
);
167 list_del(&hdl
->list
);
168 spin_unlock_irqrestore(&i40iw_handler_lock
, flags
);
173 * i40iw_enable_intr - set up device interrupts
174 * @dev: hardware control device structure
175 * @msix_id: id of the interrupt to be enabled
177 static void i40iw_enable_intr(struct i40iw_sc_dev
*dev
, u32 msix_id
)
181 val
= I40E_PFINT_DYN_CTLN_INTENA_MASK
|
182 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK
|
183 (3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT
);
185 i40iw_wr32(dev
->hw
, I40E_PFINT_DYN_CTLN(msix_id
- 1), val
);
187 i40iw_wr32(dev
->hw
, I40E_VFINT_DYN_CTLN1(msix_id
- 1), val
);
191 * i40iw_dpc - tasklet for aeq and ceq 0
192 * @data: iwarp device
194 static void i40iw_dpc(unsigned long data
)
196 struct i40iw_device
*iwdev
= (struct i40iw_device
*)data
;
198 if (iwdev
->msix_shared
)
199 i40iw_process_ceq(iwdev
, iwdev
->ceqlist
);
200 i40iw_process_aeq(iwdev
);
201 i40iw_enable_intr(&iwdev
->sc_dev
, iwdev
->iw_msixtbl
[0].idx
);
205 * i40iw_ceq_dpc - dpc handler for CEQ
206 * @data: data points to CEQ
208 static void i40iw_ceq_dpc(unsigned long data
)
210 struct i40iw_ceq
*iwceq
= (struct i40iw_ceq
*)data
;
211 struct i40iw_device
*iwdev
= iwceq
->iwdev
;
213 i40iw_process_ceq(iwdev
, iwceq
);
214 i40iw_enable_intr(&iwdev
->sc_dev
, iwceq
->msix_idx
);
218 * i40iw_irq_handler - interrupt handler for aeq and ceq0
219 * @irq: Interrupt request number
220 * @data: iwarp device
222 static irqreturn_t
i40iw_irq_handler(int irq
, void *data
)
224 struct i40iw_device
*iwdev
= (struct i40iw_device
*)data
;
226 tasklet_schedule(&iwdev
->dpc_tasklet
);
231 * i40iw_destroy_cqp - destroy control qp
232 * @iwdev: iwarp device
233 * @create_done: 1 if cqp create poll was success
235 * Issue destroy cqp request and
236 * free the resources associated with the cqp
238 static void i40iw_destroy_cqp(struct i40iw_device
*iwdev
, bool free_hwcqp
)
240 enum i40iw_status_code status
= 0;
241 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
242 struct i40iw_cqp
*cqp
= &iwdev
->cqp
;
244 if (free_hwcqp
&& dev
->cqp_ops
->cqp_destroy
)
245 status
= dev
->cqp_ops
->cqp_destroy(dev
->cqp
);
247 i40iw_pr_err("destroy cqp failed");
249 i40iw_free_dma_mem(dev
->hw
, &cqp
->sq
);
250 kfree(cqp
->scratch_array
);
251 iwdev
->cqp
.scratch_array
= NULL
;
253 kfree(cqp
->cqp_requests
);
254 cqp
->cqp_requests
= NULL
;
258 * i40iw_disable_irqs - disable device interrupts
259 * @dev: hardware control device structure
260 * @msic_vec: msix vector to disable irq
261 * @dev_id: parameter to pass to free_irq (used during irq setup)
263 * The function is called when destroying aeq/ceq
265 static void i40iw_disable_irq(struct i40iw_sc_dev
*dev
,
266 struct i40iw_msix_vector
*msix_vec
,
270 i40iw_wr32(dev
->hw
, I40E_PFINT_DYN_CTLN(msix_vec
->idx
- 1), 0);
272 i40iw_wr32(dev
->hw
, I40E_VFINT_DYN_CTLN1(msix_vec
->idx
- 1), 0);
273 free_irq(msix_vec
->irq
, dev_id
);
277 * i40iw_destroy_aeq - destroy aeq
278 * @iwdev: iwarp device
279 * @reset: true if called before reset
281 * Issue a destroy aeq request and
282 * free the resources associated with the aeq
283 * The function is called during driver unload
285 static void i40iw_destroy_aeq(struct i40iw_device
*iwdev
, bool reset
)
287 enum i40iw_status_code status
= I40IW_ERR_NOT_READY
;
288 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
289 struct i40iw_aeq
*aeq
= &iwdev
->aeq
;
291 if (!iwdev
->msix_shared
)
292 i40iw_disable_irq(dev
, iwdev
->iw_msixtbl
, (void *)iwdev
);
296 if (!dev
->aeq_ops
->aeq_destroy(&aeq
->sc_aeq
, 0, 1))
297 status
= dev
->aeq_ops
->aeq_destroy_done(&aeq
->sc_aeq
);
299 i40iw_pr_err("destroy aeq failed %d\n", status
);
302 i40iw_free_dma_mem(dev
->hw
, &aeq
->mem
);
306 * i40iw_destroy_ceq - destroy ceq
307 * @iwdev: iwarp device
308 * @iwceq: ceq to be destroyed
309 * @reset: true if called before reset
311 * Issue a destroy ceq request and
312 * free the resources associated with the ceq
314 static void i40iw_destroy_ceq(struct i40iw_device
*iwdev
,
315 struct i40iw_ceq
*iwceq
,
318 enum i40iw_status_code status
;
319 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
324 status
= dev
->ceq_ops
->ceq_destroy(&iwceq
->sc_ceq
, 0, 1);
326 i40iw_pr_err("ceq destroy command failed %d\n", status
);
330 status
= dev
->ceq_ops
->cceq_destroy_done(&iwceq
->sc_ceq
);
332 i40iw_pr_err("ceq destroy completion failed %d\n", status
);
334 i40iw_free_dma_mem(dev
->hw
, &iwceq
->mem
);
338 * i40iw_dele_ceqs - destroy all ceq's
339 * @iwdev: iwarp device
340 * @reset: true if called before reset
342 * Go through all of the device ceq's and for each ceq
343 * disable the ceq interrupt and destroy the ceq
345 static void i40iw_dele_ceqs(struct i40iw_device
*iwdev
, bool reset
)
348 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
349 struct i40iw_ceq
*iwceq
= iwdev
->ceqlist
;
350 struct i40iw_msix_vector
*msix_vec
= iwdev
->iw_msixtbl
;
352 if (iwdev
->msix_shared
) {
353 i40iw_disable_irq(dev
, msix_vec
, (void *)iwdev
);
354 i40iw_destroy_ceq(iwdev
, iwceq
, reset
);
359 for (msix_vec
++; i
< iwdev
->ceqs_count
; i
++, msix_vec
++, iwceq
++) {
360 i40iw_disable_irq(dev
, msix_vec
, (void *)iwceq
);
361 i40iw_destroy_ceq(iwdev
, iwceq
, reset
);
366 * i40iw_destroy_ccq - destroy control cq
367 * @iwdev: iwarp device
368 * @reset: true if called before reset
370 * Issue destroy ccq request and
371 * free the resources associated with the ccq
373 static void i40iw_destroy_ccq(struct i40iw_device
*iwdev
, bool reset
)
375 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
376 struct i40iw_ccq
*ccq
= &iwdev
->ccq
;
377 enum i40iw_status_code status
= 0;
380 status
= dev
->ccq_ops
->ccq_destroy(dev
->ccq
, 0, true);
382 i40iw_pr_err("ccq destroy failed %d\n", status
);
383 i40iw_free_dma_mem(dev
->hw
, &ccq
->mem_cq
);
386 /* types of hmc objects */
387 static enum i40iw_hmc_rsrc_type iw_hmc_obj_types
[] = {
392 I40IW_HMC_IW_APBVT_ENTRY
,
402 * i40iw_close_hmc_objects_type - delete hmc objects of a given type
403 * @iwdev: iwarp device
404 * @obj_type: the hmc object type to be deleted
405 * @is_pf: true if the function is PF otherwise false
406 * @reset: true if called before reset
408 static void i40iw_close_hmc_objects_type(struct i40iw_sc_dev
*dev
,
409 enum i40iw_hmc_rsrc_type obj_type
,
410 struct i40iw_hmc_info
*hmc_info
,
414 struct i40iw_hmc_del_obj_info info
;
416 memset(&info
, 0, sizeof(info
));
417 info
.hmc_info
= hmc_info
;
418 info
.rsrc_type
= obj_type
;
419 info
.count
= hmc_info
->hmc_obj
[obj_type
].cnt
;
421 if (dev
->hmc_ops
->del_hmc_object(dev
, &info
, reset
))
422 i40iw_pr_err("del obj of type %d failed\n", obj_type
);
426 * i40iw_del_hmc_objects - remove all device hmc objects
428 * @hmc_info: hmc_info to free
429 * @is_pf: true if hmc_info belongs to PF, not vf nor allocated
430 * by PF on behalf of VF
431 * @reset: true if called before reset
433 static void i40iw_del_hmc_objects(struct i40iw_sc_dev
*dev
,
434 struct i40iw_hmc_info
*hmc_info
,
440 for (i
= 0; i
< IW_HMC_OBJ_TYPE_NUM
; i
++)
441 i40iw_close_hmc_objects_type(dev
, iw_hmc_obj_types
[i
], hmc_info
, is_pf
, reset
);
445 * i40iw_ceq_handler - interrupt handler for ceq
448 static irqreturn_t
i40iw_ceq_handler(int irq
, void *data
)
450 struct i40iw_ceq
*iwceq
= (struct i40iw_ceq
*)data
;
452 if (iwceq
->irq
!= irq
)
453 i40iw_pr_err("expected irq = %d received irq = %d\n", iwceq
->irq
, irq
);
454 tasklet_schedule(&iwceq
->dpc_tasklet
);
459 * i40iw_create_hmc_obj_type - create hmc object of a given type
460 * @dev: hardware control device structure
461 * @info: information for the hmc object to create
463 static enum i40iw_status_code
i40iw_create_hmc_obj_type(struct i40iw_sc_dev
*dev
,
464 struct i40iw_hmc_create_obj_info
*info
)
466 return dev
->hmc_ops
->create_hmc_object(dev
, info
);
470 * i40iw_create_hmc_objs - create all hmc objects for the device
471 * @iwdev: iwarp device
472 * @is_pf: true if the function is PF otherwise false
474 * Create the device hmc objects and allocate hmc pages
475 * Return 0 if successful, otherwise clean up and return error
477 static enum i40iw_status_code
i40iw_create_hmc_objs(struct i40iw_device
*iwdev
,
480 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
481 struct i40iw_hmc_create_obj_info info
;
482 enum i40iw_status_code status
;
485 memset(&info
, 0, sizeof(info
));
486 info
.hmc_info
= dev
->hmc_info
;
488 info
.entry_type
= iwdev
->sd_type
;
489 for (i
= 0; i
< IW_HMC_OBJ_TYPE_NUM
; i
++) {
490 info
.rsrc_type
= iw_hmc_obj_types
[i
];
491 info
.count
= dev
->hmc_info
->hmc_obj
[info
.rsrc_type
].cnt
;
492 status
= i40iw_create_hmc_obj_type(dev
, &info
);
494 i40iw_pr_err("create obj type %d status = %d\n",
495 iw_hmc_obj_types
[i
], status
);
500 return (dev
->cqp_misc_ops
->static_hmc_pages_allocated(dev
->cqp
, 0,
506 /* destroy the hmc objects of a given type */
507 i40iw_close_hmc_objects_type(dev
,
517 * i40iw_obj_aligned_mem - get aligned memory from device allocated memory
518 * @iwdev: iwarp device
519 * @memptr: points to the memory addresses
520 * @size: size of memory needed
521 * @mask: mask for the aligned memory
523 * Get aligned memory of the requested size and
524 * update the memptr to point to the new aligned memory
525 * Return 0 if successful, otherwise return no memory error
527 enum i40iw_status_code
i40iw_obj_aligned_mem(struct i40iw_device
*iwdev
,
528 struct i40iw_dma_mem
*memptr
,
532 unsigned long va
, newva
;
535 va
= (unsigned long)iwdev
->obj_next
.va
;
538 newva
= ALIGN(va
, (mask
+ 1));
540 memptr
->va
= (u8
*)va
+ extra
;
541 memptr
->pa
= iwdev
->obj_next
.pa
+ extra
;
543 if ((memptr
->va
+ size
) > (iwdev
->obj_mem
.va
+ iwdev
->obj_mem
.size
))
544 return I40IW_ERR_NO_MEMORY
;
546 iwdev
->obj_next
.va
= memptr
->va
+ size
;
547 iwdev
->obj_next
.pa
= memptr
->pa
+ size
;
552 * i40iw_create_cqp - create control qp
553 * @iwdev: iwarp device
555 * Return 0, if the cqp and all the resources associated with it
556 * are successfully created, otherwise return error
558 static enum i40iw_status_code
i40iw_create_cqp(struct i40iw_device
*iwdev
)
560 enum i40iw_status_code status
;
561 u32 sqsize
= I40IW_CQP_SW_SQSIZE_2048
;
562 struct i40iw_dma_mem mem
;
563 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
564 struct i40iw_cqp_init_info cqp_init_info
;
565 struct i40iw_cqp
*cqp
= &iwdev
->cqp
;
566 u16 maj_err
, min_err
;
569 cqp
->cqp_requests
= kcalloc(sqsize
, sizeof(*cqp
->cqp_requests
), GFP_KERNEL
);
570 if (!cqp
->cqp_requests
)
571 return I40IW_ERR_NO_MEMORY
;
572 cqp
->scratch_array
= kcalloc(sqsize
, sizeof(*cqp
->scratch_array
), GFP_KERNEL
);
573 if (!cqp
->scratch_array
) {
574 kfree(cqp
->cqp_requests
);
575 return I40IW_ERR_NO_MEMORY
;
577 dev
->cqp
= &cqp
->sc_cqp
;
579 memset(&cqp_init_info
, 0, sizeof(cqp_init_info
));
580 status
= i40iw_allocate_dma_mem(dev
->hw
, &cqp
->sq
,
581 (sizeof(struct i40iw_cqp_sq_wqe
) * sqsize
),
582 I40IW_CQP_ALIGNMENT
);
585 status
= i40iw_obj_aligned_mem(iwdev
, &mem
, sizeof(struct i40iw_cqp_ctx
),
586 I40IW_HOST_CTX_ALIGNMENT_MASK
);
589 dev
->cqp
->host_ctx_pa
= mem
.pa
;
590 dev
->cqp
->host_ctx
= mem
.va
;
591 /* populate the cqp init info */
592 cqp_init_info
.dev
= dev
;
593 cqp_init_info
.sq_size
= sqsize
;
594 cqp_init_info
.sq
= cqp
->sq
.va
;
595 cqp_init_info
.sq_pa
= cqp
->sq
.pa
;
596 cqp_init_info
.host_ctx_pa
= mem
.pa
;
597 cqp_init_info
.host_ctx
= mem
.va
;
598 cqp_init_info
.hmc_profile
= iwdev
->resource_profile
;
599 cqp_init_info
.enabled_vf_count
= iwdev
->max_rdma_vfs
;
600 cqp_init_info
.scratch_array
= cqp
->scratch_array
;
601 status
= dev
->cqp_ops
->cqp_init(dev
->cqp
, &cqp_init_info
);
603 i40iw_pr_err("cqp init status %d maj_err %d min_err %d\n",
604 status
, maj_err
, min_err
);
607 status
= dev
->cqp_ops
->cqp_create(dev
->cqp
, true, &maj_err
, &min_err
);
609 i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n",
610 status
, maj_err
, min_err
);
613 spin_lock_init(&cqp
->req_lock
);
614 INIT_LIST_HEAD(&cqp
->cqp_avail_reqs
);
615 INIT_LIST_HEAD(&cqp
->cqp_pending_reqs
);
616 /* init the waitq of the cqp_requests and add them to the list */
617 for (i
= 0; i
< I40IW_CQP_SW_SQSIZE_2048
; i
++) {
618 init_waitqueue_head(&cqp
->cqp_requests
[i
].waitq
);
619 list_add_tail(&cqp
->cqp_requests
[i
].list
, &cqp
->cqp_avail_reqs
);
623 /* clean up the created resources */
624 i40iw_destroy_cqp(iwdev
, false);
629 * i40iw_create_ccq - create control cq
630 * @iwdev: iwarp device
632 * Return 0, if the ccq and the resources associated with it
633 * are successfully created, otherwise return error
635 static enum i40iw_status_code
i40iw_create_ccq(struct i40iw_device
*iwdev
)
637 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
638 struct i40iw_dma_mem mem
;
639 enum i40iw_status_code status
;
640 struct i40iw_ccq_init_info info
;
641 struct i40iw_ccq
*ccq
= &iwdev
->ccq
;
643 memset(&info
, 0, sizeof(info
));
644 dev
->ccq
= &ccq
->sc_cq
;
647 ccq
->shadow_area
.size
= sizeof(struct i40iw_cq_shadow_area
);
648 ccq
->mem_cq
.size
= sizeof(struct i40iw_cqe
) * IW_CCQ_SIZE
;
649 status
= i40iw_allocate_dma_mem(dev
->hw
, &ccq
->mem_cq
,
650 ccq
->mem_cq
.size
, I40IW_CQ0_ALIGNMENT
);
653 status
= i40iw_obj_aligned_mem(iwdev
, &mem
, ccq
->shadow_area
.size
,
654 I40IW_SHADOWAREA_MASK
);
657 ccq
->sc_cq
.back_cq
= (void *)ccq
;
658 /* populate the ccq init info */
659 info
.cq_base
= ccq
->mem_cq
.va
;
660 info
.cq_pa
= ccq
->mem_cq
.pa
;
661 info
.num_elem
= IW_CCQ_SIZE
;
662 info
.shadow_area
= mem
.va
;
663 info
.shadow_area_pa
= mem
.pa
;
664 info
.ceqe_mask
= false;
665 info
.ceq_id_valid
= true;
666 info
.shadow_read_threshold
= 16;
667 status
= dev
->ccq_ops
->ccq_init(dev
->ccq
, &info
);
669 status
= dev
->ccq_ops
->ccq_create(dev
->ccq
, 0, true, true);
672 i40iw_free_dma_mem(dev
->hw
, &ccq
->mem_cq
);
677 * i40iw_configure_ceq_vector - set up the msix interrupt vector for ceq
678 * @iwdev: iwarp device
679 * @msix_vec: interrupt vector information
680 * @iwceq: ceq associated with the vector
681 * @ceq_id: the id number of the iwceq
683 * Allocate interrupt resources and enable irq handling
684 * Return 0 if successful, otherwise return error
686 static enum i40iw_status_code
i40iw_configure_ceq_vector(struct i40iw_device
*iwdev
,
687 struct i40iw_ceq
*iwceq
,
689 struct i40iw_msix_vector
*msix_vec
)
691 enum i40iw_status_code status
;
693 if (iwdev
->msix_shared
&& !ceq_id
) {
694 tasklet_init(&iwdev
->dpc_tasklet
, i40iw_dpc
, (unsigned long)iwdev
);
695 status
= request_irq(msix_vec
->irq
, i40iw_irq_handler
, 0, "AEQCEQ", iwdev
);
697 tasklet_init(&iwceq
->dpc_tasklet
, i40iw_ceq_dpc
, (unsigned long)iwceq
);
698 status
= request_irq(msix_vec
->irq
, i40iw_ceq_handler
, 0, "CEQ", iwceq
);
702 i40iw_pr_err("ceq irq config fail\n");
703 return I40IW_ERR_CONFIG
;
705 msix_vec
->ceq_id
= ceq_id
;
706 msix_vec
->cpu_affinity
= 0;
712 * i40iw_create_ceq - create completion event queue
713 * @iwdev: iwarp device
714 * @iwceq: pointer to the ceq resources to be created
715 * @ceq_id: the id number of the iwceq
717 * Return 0, if the ceq and the resources associated with it
718 * are successfully created, otherwise return error
720 static enum i40iw_status_code
i40iw_create_ceq(struct i40iw_device
*iwdev
,
721 struct i40iw_ceq
*iwceq
,
724 enum i40iw_status_code status
;
725 struct i40iw_ceq_init_info info
;
726 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
729 memset(&info
, 0, sizeof(info
));
730 info
.ceq_id
= ceq_id
;
731 iwceq
->iwdev
= iwdev
;
732 iwceq
->mem
.size
= sizeof(struct i40iw_ceqe
) *
733 iwdev
->sc_dev
.hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].cnt
;
734 status
= i40iw_allocate_dma_mem(dev
->hw
, &iwceq
->mem
, iwceq
->mem
.size
,
735 I40IW_CEQ_ALIGNMENT
);
738 info
.ceq_id
= ceq_id
;
739 info
.ceqe_base
= iwceq
->mem
.va
;
740 info
.ceqe_pa
= iwceq
->mem
.pa
;
742 info
.elem_cnt
= iwdev
->sc_dev
.hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].cnt
;
743 iwceq
->sc_ceq
.ceq_id
= ceq_id
;
745 scratch
= (uintptr_t)&iwdev
->cqp
.sc_cqp
;
746 status
= dev
->ceq_ops
->ceq_init(&iwceq
->sc_ceq
, &info
);
748 status
= dev
->ceq_ops
->cceq_create(&iwceq
->sc_ceq
, scratch
);
752 i40iw_free_dma_mem(dev
->hw
, &iwceq
->mem
);
756 void i40iw_request_reset(struct i40iw_device
*iwdev
)
758 struct i40e_info
*ldev
= iwdev
->ldev
;
760 ldev
->ops
->request_reset(ldev
, iwdev
->client
, 1);
764 * i40iw_setup_ceqs - manage the device ceq's and their interrupt resources
765 * @iwdev: iwarp device
766 * @ldev: i40e lan device
768 * Allocate a list for all device completion event queues
769 * Create the ceq's and configure their msix interrupt vectors
770 * Return 0, if at least one ceq is successfully set up, otherwise return error
772 static enum i40iw_status_code
i40iw_setup_ceqs(struct i40iw_device
*iwdev
,
773 struct i40e_info
*ldev
)
777 struct i40iw_ceq
*iwceq
;
778 struct i40iw_msix_vector
*msix_vec
;
779 enum i40iw_status_code status
= 0;
782 if (ldev
&& ldev
->ops
&& ldev
->ops
->setup_qvlist
) {
783 status
= ldev
->ops
->setup_qvlist(ldev
, &i40iw_client
,
788 status
= I40IW_ERR_BAD_PTR
;
792 num_ceqs
= min(iwdev
->msix_count
, iwdev
->sc_dev
.hmc_fpm_misc
.max_ceqs
);
793 iwdev
->ceqlist
= kcalloc(num_ceqs
, sizeof(*iwdev
->ceqlist
), GFP_KERNEL
);
794 if (!iwdev
->ceqlist
) {
795 status
= I40IW_ERR_NO_MEMORY
;
798 i
= (iwdev
->msix_shared
) ? 0 : 1;
799 for (ceq_id
= 0; i
< num_ceqs
; i
++, ceq_id
++) {
800 iwceq
= &iwdev
->ceqlist
[ceq_id
];
801 status
= i40iw_create_ceq(iwdev
, iwceq
, ceq_id
);
803 i40iw_pr_err("create ceq status = %d\n", status
);
807 msix_vec
= &iwdev
->iw_msixtbl
[i
];
808 iwceq
->irq
= msix_vec
->irq
;
809 iwceq
->msix_idx
= msix_vec
->idx
;
810 status
= i40iw_configure_ceq_vector(iwdev
, iwceq
, ceq_id
, msix_vec
);
812 i40iw_destroy_ceq(iwdev
, iwceq
, false);
815 i40iw_enable_intr(&iwdev
->sc_dev
, msix_vec
->idx
);
821 if (!iwdev
->ceqs_count
) {
822 kfree(iwdev
->ceqlist
);
823 iwdev
->ceqlist
= NULL
;
832 * i40iw_configure_aeq_vector - set up the msix vector for aeq
833 * @iwdev: iwarp device
835 * Allocate interrupt resources and enable irq handling
836 * Return 0 if successful, otherwise return error
838 static enum i40iw_status_code
i40iw_configure_aeq_vector(struct i40iw_device
*iwdev
)
840 struct i40iw_msix_vector
*msix_vec
= iwdev
->iw_msixtbl
;
843 if (!iwdev
->msix_shared
) {
844 tasklet_init(&iwdev
->dpc_tasklet
, i40iw_dpc
, (unsigned long)iwdev
);
845 ret
= request_irq(msix_vec
->irq
, i40iw_irq_handler
, 0, "i40iw", iwdev
);
848 i40iw_pr_err("aeq irq config fail\n");
849 return I40IW_ERR_CONFIG
;
856 * i40iw_create_aeq - create async event queue
857 * @iwdev: iwarp device
859 * Return 0, if the aeq and the resources associated with it
860 * are successfully created, otherwise return error
862 static enum i40iw_status_code
i40iw_create_aeq(struct i40iw_device
*iwdev
)
864 enum i40iw_status_code status
;
865 struct i40iw_aeq_init_info info
;
866 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
867 struct i40iw_aeq
*aeq
= &iwdev
->aeq
;
871 aeq_size
= 2 * iwdev
->sc_dev
.hmc_info
->hmc_obj
[I40IW_HMC_IW_QP
].cnt
+
872 iwdev
->sc_dev
.hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].cnt
;
873 memset(&info
, 0, sizeof(info
));
874 aeq
->mem
.size
= sizeof(struct i40iw_sc_aeqe
) * aeq_size
;
875 status
= i40iw_allocate_dma_mem(dev
->hw
, &aeq
->mem
, aeq
->mem
.size
,
876 I40IW_AEQ_ALIGNMENT
);
880 info
.aeqe_base
= aeq
->mem
.va
;
881 info
.aeq_elem_pa
= aeq
->mem
.pa
;
882 info
.elem_cnt
= aeq_size
;
884 status
= dev
->aeq_ops
->aeq_init(&aeq
->sc_aeq
, &info
);
887 status
= dev
->aeq_ops
->aeq_create(&aeq
->sc_aeq
, scratch
, 1);
889 status
= dev
->aeq_ops
->aeq_create_done(&aeq
->sc_aeq
);
892 i40iw_free_dma_mem(dev
->hw
, &aeq
->mem
);
897 * i40iw_setup_aeq - set up the device aeq
898 * @iwdev: iwarp device
900 * Create the aeq and configure its msix interrupt vector
901 * Return 0 if successful, otherwise return error
903 static enum i40iw_status_code
i40iw_setup_aeq(struct i40iw_device
*iwdev
)
905 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
906 enum i40iw_status_code status
;
908 status
= i40iw_create_aeq(iwdev
);
912 status
= i40iw_configure_aeq_vector(iwdev
);
914 i40iw_destroy_aeq(iwdev
, false);
918 if (!iwdev
->msix_shared
)
919 i40iw_enable_intr(dev
, iwdev
->iw_msixtbl
[0].idx
);
924 * i40iw_initialize_ilq - create iwarp local queue for cm
925 * @iwdev: iwarp device
927 * Return 0 if successful, otherwise return error
929 static enum i40iw_status_code
i40iw_initialize_ilq(struct i40iw_device
*iwdev
)
931 struct i40iw_puda_rsrc_info info
;
932 enum i40iw_status_code status
;
934 info
.type
= I40IW_PUDA_RSRC_TYPE_ILQ
;
941 info
.buf_size
= 1024;
942 info
.tx_buf_cnt
= 16384;
943 info
.mss
= iwdev
->mss
;
944 info
.receive
= i40iw_receive_ilq
;
945 info
.xmit_complete
= i40iw_free_sqbuf
;
946 status
= i40iw_puda_create_rsrc(&iwdev
->sc_dev
, &info
);
948 i40iw_pr_err("ilq create fail\n");
953 * i40iw_initialize_ieq - create iwarp exception queue
954 * @iwdev: iwarp device
956 * Return 0 if successful, otherwise return error
958 static enum i40iw_status_code
i40iw_initialize_ieq(struct i40iw_device
*iwdev
)
960 struct i40iw_puda_rsrc_info info
;
961 enum i40iw_status_code status
;
963 info
.type
= I40IW_PUDA_RSRC_TYPE_IEQ
;
965 info
.qp_id
= iwdev
->sc_dev
.exception_lan_queue
;
970 info
.buf_size
= 2048;
971 info
.mss
= iwdev
->mss
;
972 info
.tx_buf_cnt
= 16384;
973 status
= i40iw_puda_create_rsrc(&iwdev
->sc_dev
, &info
);
975 i40iw_pr_err("ieq create fail\n");
980 * i40iw_hmc_setup - create hmc objects for the device
981 * @iwdev: iwarp device
983 * Set up the device private memory space for the number and size of
984 * the hmc objects and create the objects
985 * Return 0 if successful, otherwise return error
987 static enum i40iw_status_code
i40iw_hmc_setup(struct i40iw_device
*iwdev
)
989 enum i40iw_status_code status
;
991 iwdev
->sd_type
= I40IW_SD_TYPE_DIRECT
;
992 status
= i40iw_config_fpm_values(&iwdev
->sc_dev
, IW_CFG_FPM_QP_COUNT
);
995 status
= i40iw_create_hmc_objs(iwdev
, true);
998 iwdev
->init_state
= HMC_OBJS_CREATED
;
1004 * i40iw_del_init_mem - deallocate memory resources
1005 * @iwdev: iwarp device
1007 static void i40iw_del_init_mem(struct i40iw_device
*iwdev
)
1009 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
1011 i40iw_free_dma_mem(&iwdev
->hw
, &iwdev
->obj_mem
);
1012 kfree(dev
->hmc_info
->sd_table
.sd_entry
);
1013 dev
->hmc_info
->sd_table
.sd_entry
= NULL
;
1014 kfree(iwdev
->mem_resources
);
1015 iwdev
->mem_resources
= NULL
;
1016 kfree(iwdev
->ceqlist
);
1017 iwdev
->ceqlist
= NULL
;
1018 kfree(iwdev
->iw_msixtbl
);
1019 iwdev
->iw_msixtbl
= NULL
;
1020 kfree(iwdev
->hmc_info_mem
);
1021 iwdev
->hmc_info_mem
= NULL
;
1025 * i40iw_del_macip_entry - remove a mac ip address entry from the hw table
1026 * @iwdev: iwarp device
1027 * @idx: the index of the mac ip address to delete
1029 static void i40iw_del_macip_entry(struct i40iw_device
*iwdev
, u8 idx
)
1031 struct i40iw_cqp
*iwcqp
= &iwdev
->cqp
;
1032 struct i40iw_cqp_request
*cqp_request
;
1033 struct cqp_commands_info
*cqp_info
;
1034 enum i40iw_status_code status
= 0;
1036 cqp_request
= i40iw_get_cqp_request(iwcqp
, true);
1038 i40iw_pr_err("cqp_request memory failed\n");
1041 cqp_info
= &cqp_request
->info
;
1042 cqp_info
->cqp_cmd
= OP_DELETE_LOCAL_MAC_IPADDR_ENTRY
;
1043 cqp_info
->post_sq
= 1;
1044 cqp_info
->in
.u
.del_local_mac_ipaddr_entry
.cqp
= &iwcqp
->sc_cqp
;
1045 cqp_info
->in
.u
.del_local_mac_ipaddr_entry
.scratch
= (uintptr_t)cqp_request
;
1046 cqp_info
->in
.u
.del_local_mac_ipaddr_entry
.entry_idx
= idx
;
1047 cqp_info
->in
.u
.del_local_mac_ipaddr_entry
.ignore_ref_count
= 0;
1048 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1050 i40iw_pr_err("CQP-OP Del MAC Ip entry fail");
1054 * i40iw_add_mac_ipaddr_entry - add a mac ip address entry to the hw table
1055 * @iwdev: iwarp device
1056 * @mac_addr: pointer to mac address
1057 * @idx: the index of the mac ip address to add
1059 static enum i40iw_status_code
i40iw_add_mac_ipaddr_entry(struct i40iw_device
*iwdev
,
1063 struct i40iw_local_mac_ipaddr_entry_info
*info
;
1064 struct i40iw_cqp
*iwcqp
= &iwdev
->cqp
;
1065 struct i40iw_cqp_request
*cqp_request
;
1066 struct cqp_commands_info
*cqp_info
;
1067 enum i40iw_status_code status
= 0;
1069 cqp_request
= i40iw_get_cqp_request(iwcqp
, true);
1071 i40iw_pr_err("cqp_request memory failed\n");
1072 return I40IW_ERR_NO_MEMORY
;
1075 cqp_info
= &cqp_request
->info
;
1077 cqp_info
->post_sq
= 1;
1078 info
= &cqp_info
->in
.u
.add_local_mac_ipaddr_entry
.info
;
1079 ether_addr_copy(info
->mac_addr
, mac_addr
);
1080 info
->entry_idx
= idx
;
1081 cqp_info
->in
.u
.add_local_mac_ipaddr_entry
.scratch
= (uintptr_t)cqp_request
;
1082 cqp_info
->cqp_cmd
= OP_ADD_LOCAL_MAC_IPADDR_ENTRY
;
1083 cqp_info
->in
.u
.add_local_mac_ipaddr_entry
.cqp
= &iwcqp
->sc_cqp
;
1084 cqp_info
->in
.u
.add_local_mac_ipaddr_entry
.scratch
= (uintptr_t)cqp_request
;
1085 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1087 i40iw_pr_err("CQP-OP Add MAC Ip entry fail");
1092 * i40iw_alloc_local_mac_ipaddr_entry - allocate a mac ip address entry
1093 * @iwdev: iwarp device
1094 * @mac_ip_tbl_idx: the index of the new mac ip address
1096 * Allocate a mac ip address entry and update the mac_ip_tbl_idx
1097 * to hold the index of the newly created mac ip address
1098 * Return 0 if successful, otherwise return error
1100 static enum i40iw_status_code
i40iw_alloc_local_mac_ipaddr_entry(struct i40iw_device
*iwdev
,
1101 u16
*mac_ip_tbl_idx
)
1103 struct i40iw_cqp
*iwcqp
= &iwdev
->cqp
;
1104 struct i40iw_cqp_request
*cqp_request
;
1105 struct cqp_commands_info
*cqp_info
;
1106 enum i40iw_status_code status
= 0;
1108 cqp_request
= i40iw_get_cqp_request(iwcqp
, true);
1110 i40iw_pr_err("cqp_request memory failed\n");
1111 return I40IW_ERR_NO_MEMORY
;
1114 /* increment refcount, because we need the cqp request ret value */
1115 atomic_inc(&cqp_request
->refcount
);
1117 cqp_info
= &cqp_request
->info
;
1118 cqp_info
->cqp_cmd
= OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY
;
1119 cqp_info
->post_sq
= 1;
1120 cqp_info
->in
.u
.alloc_local_mac_ipaddr_entry
.cqp
= &iwcqp
->sc_cqp
;
1121 cqp_info
->in
.u
.alloc_local_mac_ipaddr_entry
.scratch
= (uintptr_t)cqp_request
;
1122 status
= i40iw_handle_cqp_op(iwdev
, cqp_request
);
1124 *mac_ip_tbl_idx
= cqp_request
->compl_info
.op_ret_val
;
1126 i40iw_pr_err("CQP-OP Alloc MAC Ip entry fail");
1127 /* decrement refcount and free the cqp request, if no longer used */
1128 i40iw_put_cqp_request(iwcqp
, cqp_request
);
1133 * i40iw_alloc_set_mac_ipaddr - set up a mac ip address table entry
1134 * @iwdev: iwarp device
1135 * @macaddr: pointer to mac address
1137 * Allocate a mac ip address entry and add it to the hw table
1138 * Return 0 if successful, otherwise return error
1140 static enum i40iw_status_code
i40iw_alloc_set_mac_ipaddr(struct i40iw_device
*iwdev
,
1143 enum i40iw_status_code status
;
1145 status
= i40iw_alloc_local_mac_ipaddr_entry(iwdev
, &iwdev
->mac_ip_table_idx
);
1147 status
= i40iw_add_mac_ipaddr_entry(iwdev
, macaddr
,
1148 (u8
)iwdev
->mac_ip_table_idx
);
1150 i40iw_del_macip_entry(iwdev
, (u8
)iwdev
->mac_ip_table_idx
);
1156 * i40iw_add_ipv6_addr - add ipv6 address to the hw arp table
1157 * @iwdev: iwarp device
1159 static void i40iw_add_ipv6_addr(struct i40iw_device
*iwdev
)
1161 struct net_device
*ip_dev
;
1162 struct inet6_dev
*idev
;
1163 struct inet6_ifaddr
*ifp
;
1164 u32 local_ipaddr6
[4];
1167 for_each_netdev_rcu(&init_net
, ip_dev
) {
1168 if ((((rdma_vlan_dev_vlan_id(ip_dev
) < 0xFFFF) &&
1169 (rdma_vlan_dev_real_dev(ip_dev
) == iwdev
->netdev
)) ||
1170 (ip_dev
== iwdev
->netdev
)) && (ip_dev
->flags
& IFF_UP
)) {
1171 idev
= __in6_dev_get(ip_dev
);
1173 i40iw_pr_err("ipv6 inet device not found\n");
1176 list_for_each_entry(ifp
, &idev
->addr_list
, if_list
) {
1177 i40iw_pr_info("IP=%pI6, vlan_id=%d, MAC=%pM\n", &ifp
->addr
,
1178 rdma_vlan_dev_vlan_id(ip_dev
), ip_dev
->dev_addr
);
1179 i40iw_copy_ip_ntohl(local_ipaddr6
,
1180 ifp
->addr
.in6_u
.u6_addr32
);
1181 i40iw_manage_arp_cache(iwdev
,
1193 * i40iw_add_ipv4_addr - add ipv4 address to the hw arp table
1194 * @iwdev: iwarp device
1196 static void i40iw_add_ipv4_addr(struct i40iw_device
*iwdev
)
1198 struct net_device
*dev
;
1199 struct in_device
*idev
;
1200 bool got_lock
= true;
1203 if (!rtnl_trylock())
1206 for_each_netdev(&init_net
, dev
) {
1207 if ((((rdma_vlan_dev_vlan_id(dev
) < 0xFFFF) &&
1208 (rdma_vlan_dev_real_dev(dev
) == iwdev
->netdev
)) ||
1209 (dev
== iwdev
->netdev
)) && (dev
->flags
& IFF_UP
)) {
1210 idev
= in_dev_get(dev
);
1212 i40iw_debug(&iwdev
->sc_dev
, I40IW_DEBUG_CM
,
1213 "IP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa
->ifa_address
,
1214 rdma_vlan_dev_vlan_id(dev
), dev
->dev_addr
);
1216 ip_addr
= ntohl(ifa
->ifa_address
);
1217 i40iw_manage_arp_cache(iwdev
,
1232 * i40iw_add_mac_ip - add mac and ip addresses
1233 * @iwdev: iwarp device
1235 * Create and add a mac ip address entry to the hw table and
1236 * ipv4/ipv6 addresses to the arp cache
1237 * Return 0 if successful, otherwise return error
1239 static enum i40iw_status_code
i40iw_add_mac_ip(struct i40iw_device
*iwdev
)
1241 struct net_device
*netdev
= iwdev
->netdev
;
1242 enum i40iw_status_code status
;
1244 status
= i40iw_alloc_set_mac_ipaddr(iwdev
, (u8
*)netdev
->dev_addr
);
1247 i40iw_add_ipv4_addr(iwdev
);
1248 i40iw_add_ipv6_addr(iwdev
);
1253 * i40iw_wait_pe_ready - Check if firmware is ready
1254 * @hw: provides access to registers
1256 static void i40iw_wait_pe_ready(struct i40iw_hw
*hw
)
1265 statusfw
= i40iw_rd32(hw
, I40E_GLPE_FWLDSTATUS
);
1266 i40iw_pr_info("[%04d] fm load status[x%04X]\n", __LINE__
, statusfw
);
1267 statuscpu0
= i40iw_rd32(hw
, I40E_GLPE_CPUSTATUS0
);
1268 i40iw_pr_info("[%04d] CSR_CQP status[x%04X]\n", __LINE__
, statuscpu0
);
1269 statuscpu1
= i40iw_rd32(hw
, I40E_GLPE_CPUSTATUS1
);
1270 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS1 status[x%04X]\n",
1271 __LINE__
, statuscpu1
);
1272 statuscpu2
= i40iw_rd32(hw
, I40E_GLPE_CPUSTATUS2
);
1273 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS2 status[x%04X]\n",
1274 __LINE__
, statuscpu2
);
1275 if ((statuscpu0
== 0x80) && (statuscpu1
== 0x80) && (statuscpu2
== 0x80))
1276 break; /* SUCCESS */
1279 } while (retrycount
< 14);
1280 i40iw_wr32(hw
, 0xb4040, 0x4C104C5);
1284 * i40iw_initialize_dev - initialize device
1285 * @iwdev: iwarp device
1286 * @ldev: lan device information
1288 * Allocate memory for the hmc objects and initialize iwdev
1289 * Return 0 if successful, otherwise clean up the resources
1292 static enum i40iw_status_code
i40iw_initialize_dev(struct i40iw_device
*iwdev
,
1293 struct i40e_info
*ldev
)
1295 enum i40iw_status_code status
;
1296 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
1297 struct i40iw_device_init_info info
;
1298 struct i40iw_dma_mem mem
;
1301 memset(&info
, 0, sizeof(info
));
1302 size
= sizeof(struct i40iw_hmc_pble_rsrc
) + sizeof(struct i40iw_hmc_info
) +
1303 (sizeof(struct i40iw_hmc_obj_info
) * I40IW_HMC_IW_MAX
);
1304 iwdev
->hmc_info_mem
= kzalloc(size
, GFP_KERNEL
);
1305 if (!iwdev
->hmc_info_mem
) {
1306 i40iw_pr_err("memory alloc fail\n");
1307 return I40IW_ERR_NO_MEMORY
;
1309 iwdev
->pble_rsrc
= (struct i40iw_hmc_pble_rsrc
*)iwdev
->hmc_info_mem
;
1310 dev
->hmc_info
= &iwdev
->hw
.hmc
;
1311 dev
->hmc_info
->hmc_obj
= (struct i40iw_hmc_obj_info
*)(iwdev
->pble_rsrc
+ 1);
1312 status
= i40iw_obj_aligned_mem(iwdev
, &mem
, I40IW_QUERY_FPM_BUF_SIZE
,
1313 I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK
);
1316 info
.fpm_query_buf_pa
= mem
.pa
;
1317 info
.fpm_query_buf
= mem
.va
;
1318 status
= i40iw_obj_aligned_mem(iwdev
, &mem
, I40IW_COMMIT_FPM_BUF_SIZE
,
1319 I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK
);
1322 info
.fpm_commit_buf_pa
= mem
.pa
;
1323 info
.fpm_commit_buf
= mem
.va
;
1324 info
.hmc_fn_id
= ldev
->fid
;
1325 info
.is_pf
= (ldev
->ftype
) ? false : true;
1326 info
.bar0
= ldev
->hw_addr
;
1327 info
.hw
= &iwdev
->hw
;
1328 info
.debug_mask
= debug
;
1329 info
.qs_handle
= ldev
->params
.qos
.prio_qos
[0].qs_handle
;
1330 info
.exception_lan_queue
= 1;
1331 info
.vchnl_send
= i40iw_virtchnl_send
;
1332 status
= i40iw_device_init(&iwdev
->sc_dev
, &info
);
1335 kfree(iwdev
->hmc_info_mem
);
1336 iwdev
->hmc_info_mem
= NULL
;
1342 * i40iw_register_notifiers - register tcp ip notifiers
1344 static void i40iw_register_notifiers(void)
1346 if (!i40iw_notifiers_registered
) {
1347 register_inetaddr_notifier(&i40iw_inetaddr_notifier
);
1348 register_inet6addr_notifier(&i40iw_inetaddr6_notifier
);
1349 register_netevent_notifier(&i40iw_net_notifier
);
1351 i40iw_notifiers_registered
++;
1355 * i40iw_save_msix_info - copy msix vector information to iwarp device
1356 * @iwdev: iwarp device
1357 * @ldev: lan device information
1359 * Allocate iwdev msix table and copy the ldev msix info to the table
1360 * Return 0 if successful, otherwise return error
1362 static enum i40iw_status_code
i40iw_save_msix_info(struct i40iw_device
*iwdev
,
1363 struct i40e_info
*ldev
)
1365 struct i40e_qvlist_info
*iw_qvlist
;
1366 struct i40e_qv_info
*iw_qvinfo
;
1371 iwdev
->msix_count
= ldev
->msix_count
;
1373 size
= sizeof(struct i40iw_msix_vector
) * iwdev
->msix_count
;
1374 size
+= sizeof(struct i40e_qvlist_info
);
1375 size
+= sizeof(struct i40e_qv_info
) * iwdev
->msix_count
- 1;
1376 iwdev
->iw_msixtbl
= kzalloc(size
, GFP_KERNEL
);
1378 if (!iwdev
->iw_msixtbl
)
1379 return I40IW_ERR_NO_MEMORY
;
1380 iwdev
->iw_qvlist
= (struct i40e_qvlist_info
*)(&iwdev
->iw_msixtbl
[iwdev
->msix_count
]);
1381 iw_qvlist
= iwdev
->iw_qvlist
;
1382 iw_qvinfo
= iw_qvlist
->qv_info
;
1383 iw_qvlist
->num_vectors
= iwdev
->msix_count
;
1384 if (iwdev
->msix_count
<= num_online_cpus())
1385 iwdev
->msix_shared
= true;
1386 for (i
= 0, ceq_idx
= 0; i
< iwdev
->msix_count
; i
++, iw_qvinfo
++) {
1387 iwdev
->iw_msixtbl
[i
].idx
= ldev
->msix_entries
[i
].entry
;
1388 iwdev
->iw_msixtbl
[i
].irq
= ldev
->msix_entries
[i
].vector
;
1390 iw_qvinfo
->aeq_idx
= 0;
1391 if (iwdev
->msix_shared
)
1392 iw_qvinfo
->ceq_idx
= ceq_idx
++;
1394 iw_qvinfo
->ceq_idx
= I40E_QUEUE_INVALID_IDX
;
1396 iw_qvinfo
->aeq_idx
= I40E_QUEUE_INVALID_IDX
;
1397 iw_qvinfo
->ceq_idx
= ceq_idx
++;
1399 iw_qvinfo
->itr_idx
= 3;
1400 iw_qvinfo
->v_idx
= iwdev
->iw_msixtbl
[i
].idx
;
1406 * i40iw_deinit_device - clean up the device resources
1407 * @iwdev: iwarp device
1408 * @reset: true if called before reset
1409 * @del_hdl: true if delete hdl entry
1411 * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6 addresses,
1412 * destroy the device queues and free the pble and the hmc objects
1414 static void i40iw_deinit_device(struct i40iw_device
*iwdev
, bool reset
, bool del_hdl
)
1416 struct i40e_info
*ldev
= iwdev
->ldev
;
1418 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
1420 i40iw_pr_info("state = %d\n", iwdev
->init_state
);
1422 switch (iwdev
->init_state
) {
1423 case RDMA_DEV_REGISTERED
:
1424 iwdev
->iw_status
= 0;
1425 i40iw_port_ibevent(iwdev
);
1426 i40iw_destroy_rdma_device(iwdev
->iwibdev
);
1428 case IP_ADDR_REGISTERED
:
1430 i40iw_del_macip_entry(iwdev
, (u8
)iwdev
->mac_ip_table_idx
);
1433 if (i40iw_notifiers_registered
> 0) {
1434 i40iw_notifiers_registered
--;
1435 unregister_netevent_notifier(&i40iw_net_notifier
);
1436 unregister_inetaddr_notifier(&i40iw_inetaddr_notifier
);
1437 unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier
);
1441 i40iw_dele_ceqs(iwdev
, reset
);
1444 i40iw_destroy_aeq(iwdev
, reset
);
1447 i40iw_puda_dele_resources(dev
, I40IW_PUDA_RSRC_TYPE_IEQ
, reset
);
1450 i40iw_puda_dele_resources(dev
, I40IW_PUDA_RSRC_TYPE_ILQ
, reset
);
1453 i40iw_destroy_ccq(iwdev
, reset
);
1455 case PBLE_CHUNK_MEM
:
1456 i40iw_destroy_pble_pool(dev
, iwdev
->pble_rsrc
);
1458 case HMC_OBJS_CREATED
:
1459 i40iw_del_hmc_objects(dev
, dev
->hmc_info
, true, reset
);
1462 i40iw_destroy_cqp(iwdev
, !reset
);
1465 i40iw_cleanup_cm_core(&iwdev
->cm_core
);
1467 i40iw_hw_stats_del_timer(dev
);
1469 i40iw_del_init_mem(iwdev
);
1474 i40iw_pr_err("bad init_state = %d\n", iwdev
->init_state
);
1479 i40iw_del_handler(i40iw_find_i40e_handler(ldev
));
1484 * i40iw_setup_init_state - set up the initial device struct
1485 * @hdl: handler for iwarp device - one per instance
1486 * @ldev: lan device information
1487 * @client: iwarp client information, provided during registration
1489 * Initialize the iwarp device and its hdl information
1490 * using the ldev and client information
1491 * Return 0 if successful, otherwise return error
1493 static enum i40iw_status_code
i40iw_setup_init_state(struct i40iw_handler
*hdl
,
1494 struct i40e_info
*ldev
,
1495 struct i40e_client
*client
)
1497 struct i40iw_device
*iwdev
= &hdl
->device
;
1498 struct i40iw_sc_dev
*dev
= &iwdev
->sc_dev
;
1499 enum i40iw_status_code status
;
1501 memcpy(&hdl
->ldev
, ldev
, sizeof(*ldev
));
1502 if (resource_profile
== 1)
1503 resource_profile
= 2;
1505 iwdev
->mpa_version
= mpa_version
;
1506 iwdev
->resource_profile
= (resource_profile
< I40IW_HMC_PROFILE_EQUAL
) ?
1507 (u8
)resource_profile
+ I40IW_HMC_PROFILE_DEFAULT
:
1508 I40IW_HMC_PROFILE_DEFAULT
;
1509 iwdev
->max_rdma_vfs
=
1510 (iwdev
->resource_profile
!= I40IW_HMC_PROFILE_DEFAULT
) ? max_rdma_vfs
: 0;
1511 iwdev
->max_enabled_vfs
= iwdev
->max_rdma_vfs
;
1512 iwdev
->netdev
= ldev
->netdev
;
1513 hdl
->client
= client
;
1514 iwdev
->mss
= (!ldev
->params
.mtu
) ? I40IW_DEFAULT_MSS
: ldev
->params
.mtu
- I40IW_MTU_TO_MSS
;
1516 iwdev
->db_start
= pci_resource_start(ldev
->pcidev
, 0) + I40IW_DB_ADDR_OFFSET
;
1518 iwdev
->db_start
= pci_resource_start(ldev
->pcidev
, 0) + I40IW_VF_DB_ADDR_OFFSET
;
1520 status
= i40iw_save_msix_info(iwdev
, ldev
);
1523 iwdev
->hw
.dev_context
= (void *)ldev
->pcidev
;
1524 iwdev
->hw
.hw_addr
= ldev
->hw_addr
;
1525 status
= i40iw_allocate_dma_mem(&iwdev
->hw
,
1526 &iwdev
->obj_mem
, 8192, 4096);
1529 iwdev
->obj_next
= iwdev
->obj_mem
;
1530 iwdev
->push_mode
= push_mode
;
1532 init_waitqueue_head(&iwdev
->vchnl_waitq
);
1533 init_waitqueue_head(&dev
->vf_reqs
);
1535 status
= i40iw_initialize_dev(iwdev
, ldev
);
1538 kfree(iwdev
->iw_msixtbl
);
1539 i40iw_free_dma_mem(dev
->hw
, &iwdev
->obj_mem
);
1540 iwdev
->iw_msixtbl
= NULL
;
1546 * i40iw_open - client interface operation open for iwarp/uda device
1547 * @ldev: lan device information
1548 * @client: iwarp client information, provided during registration
1550 * Called by the lan driver during the processing of client register
1551 * Create device resources, set up queues, pble and hmc objects and
1552 * register the device with the ib verbs interface
1553 * Return 0 if successful, otherwise return error
1555 static int i40iw_open(struct i40e_info
*ldev
, struct i40e_client
*client
)
1557 struct i40iw_device
*iwdev
;
1558 struct i40iw_sc_dev
*dev
;
1559 enum i40iw_status_code status
;
1560 struct i40iw_handler
*hdl
;
1562 hdl
= kzalloc(sizeof(*hdl
), GFP_KERNEL
);
1565 iwdev
= &hdl
->device
;
1567 dev
= &iwdev
->sc_dev
;
1568 i40iw_setup_cm_core(iwdev
);
1570 dev
->back_dev
= (void *)iwdev
;
1571 iwdev
->ldev
= &hdl
->ldev
;
1572 iwdev
->client
= client
;
1573 mutex_init(&iwdev
->pbl_mutex
);
1574 i40iw_add_handler(hdl
);
1577 status
= i40iw_setup_init_state(hdl
, ldev
, client
);
1580 iwdev
->init_state
= INITIAL_STATE
;
1582 i40iw_wait_pe_ready(dev
->hw
);
1583 status
= i40iw_create_cqp(iwdev
);
1586 iwdev
->init_state
= CQP_CREATED
;
1587 status
= i40iw_hmc_setup(iwdev
);
1590 status
= i40iw_create_ccq(iwdev
);
1593 iwdev
->init_state
= CCQ_CREATED
;
1594 status
= i40iw_initialize_ilq(iwdev
);
1597 iwdev
->init_state
= ILQ_CREATED
;
1598 status
= i40iw_initialize_ieq(iwdev
);
1601 iwdev
->init_state
= IEQ_CREATED
;
1602 status
= i40iw_setup_aeq(iwdev
);
1605 iwdev
->init_state
= AEQ_CREATED
;
1606 status
= i40iw_setup_ceqs(iwdev
, ldev
);
1609 iwdev
->init_state
= CEQ_CREATED
;
1610 status
= i40iw_initialize_hw_resources(iwdev
);
1613 dev
->ccq_ops
->ccq_arm(dev
->ccq
);
1614 status
= i40iw_hmc_init_pble(&iwdev
->sc_dev
, iwdev
->pble_rsrc
);
1617 iwdev
->virtchnl_wq
= create_singlethread_workqueue("iwvch");
1618 i40iw_register_notifiers();
1619 iwdev
->init_state
= INET_NOTIFIER
;
1620 status
= i40iw_add_mac_ip(iwdev
);
1623 iwdev
->init_state
= IP_ADDR_REGISTERED
;
1624 if (i40iw_register_rdma_device(iwdev
)) {
1625 i40iw_pr_err("register rdma device fail\n");
1629 iwdev
->init_state
= RDMA_DEV_REGISTERED
;
1630 iwdev
->iw_status
= 1;
1631 i40iw_port_ibevent(iwdev
);
1632 i40iw_pr_info("i40iw_open completed\n");
1636 i40iw_pr_err("status = %d last completion = %d\n", status
, iwdev
->init_state
);
1637 i40iw_deinit_device(iwdev
, false, false);
1642 * i40iw_l2param_change : handle qs handles for qos and mss change
1643 * @ldev: lan device information
1644 * @client: client for paramater change
1645 * @params: new parameters from L2
1647 static void i40iw_l2param_change(struct i40e_info
*ldev
,
1648 struct i40e_client
*client
,
1649 struct i40e_params
*params
)
1651 struct i40iw_handler
*hdl
;
1652 struct i40iw_device
*iwdev
;
1654 hdl
= i40iw_find_i40e_handler(ldev
);
1658 iwdev
= &hdl
->device
;
1660 iwdev
->mss
= params
->mtu
- I40IW_MTU_TO_MSS
;
1664 * i40iw_close - client interface operation close for iwarp/uda device
1665 * @ldev: lan device information
1666 * @client: client to close
1668 * Called by the lan driver during the processing of client unregister
1669 * Destroy and clean up the driver resources
1671 static void i40iw_close(struct i40e_info
*ldev
, struct i40e_client
*client
, bool reset
)
1673 struct i40iw_device
*iwdev
;
1674 struct i40iw_handler
*hdl
;
1676 hdl
= i40iw_find_i40e_handler(ldev
);
1680 iwdev
= &hdl
->device
;
1681 destroy_workqueue(iwdev
->virtchnl_wq
);
1682 i40iw_deinit_device(iwdev
, reset
, true);
1686 * i40iw_vf_reset - process VF reset
1687 * @ldev: lan device information
1688 * @client: client interface instance
1689 * @vf_id: virtual function id
1691 * Called when a VF is reset by the PF
1692 * Destroy and clean up the VF resources
1694 static void i40iw_vf_reset(struct i40e_info
*ldev
, struct i40e_client
*client
, u32 vf_id
)
1696 struct i40iw_handler
*hdl
;
1697 struct i40iw_sc_dev
*dev
;
1698 struct i40iw_hmc_fcn_info hmc_fcn_info
;
1699 struct i40iw_virt_mem vf_dev_mem
;
1700 struct i40iw_vfdev
*tmp_vfdev
;
1702 unsigned long flags
;
1704 hdl
= i40iw_find_i40e_handler(ldev
);
1708 dev
= &hdl
->device
.sc_dev
;
1710 for (i
= 0; i
< I40IW_MAX_PE_ENABLED_VF_COUNT
; i
++) {
1711 if (!dev
->vf_dev
[i
] || (dev
->vf_dev
[i
]->vf_id
!= vf_id
))
1713 /* free all resources allocated on behalf of vf */
1714 tmp_vfdev
= dev
->vf_dev
[i
];
1715 spin_lock_irqsave(&dev
->dev_pestat
.stats_lock
, flags
);
1716 dev
->vf_dev
[i
] = NULL
;
1717 spin_unlock_irqrestore(&dev
->dev_pestat
.stats_lock
, flags
);
1718 i40iw_del_hmc_objects(dev
, &tmp_vfdev
->hmc_info
, false, false);
1719 /* remove vf hmc function */
1720 memset(&hmc_fcn_info
, 0, sizeof(hmc_fcn_info
));
1721 hmc_fcn_info
.vf_id
= vf_id
;
1722 hmc_fcn_info
.iw_vf_idx
= tmp_vfdev
->iw_vf_idx
;
1723 hmc_fcn_info
.free_fcn
= true;
1724 i40iw_cqp_manage_hmc_fcn_cmd(dev
, &hmc_fcn_info
);
1726 vf_dev_mem
.va
= tmp_vfdev
;
1727 vf_dev_mem
.size
= sizeof(struct i40iw_vfdev
) +
1728 sizeof(struct i40iw_hmc_obj_info
) * I40IW_HMC_IW_MAX
;
1729 i40iw_free_virt_mem(dev
->hw
, &vf_dev_mem
);
1735 * i40iw_vf_enable - enable a number of VFs
1736 * @ldev: lan device information
1737 * @client: client interface instance
1738 * @num_vfs: number of VFs for the PF
1740 * Called when the number of VFs changes
1742 static void i40iw_vf_enable(struct i40e_info
*ldev
,
1743 struct i40e_client
*client
,
1746 struct i40iw_handler
*hdl
;
1748 hdl
= i40iw_find_i40e_handler(ldev
);
1752 if (num_vfs
> I40IW_MAX_PE_ENABLED_VF_COUNT
)
1753 hdl
->device
.max_enabled_vfs
= I40IW_MAX_PE_ENABLED_VF_COUNT
;
1755 hdl
->device
.max_enabled_vfs
= num_vfs
;
1759 * i40iw_vf_capable - check if VF capable
1760 * @ldev: lan device information
1761 * @client: client interface instance
1762 * @vf_id: virtual function id
1764 * Return 1 if a VF slot is available or if VF is already RDMA enabled
1765 * Return 0 otherwise
1767 static int i40iw_vf_capable(struct i40e_info
*ldev
,
1768 struct i40e_client
*client
,
1771 struct i40iw_handler
*hdl
;
1772 struct i40iw_sc_dev
*dev
;
1775 hdl
= i40iw_find_i40e_handler(ldev
);
1779 dev
= &hdl
->device
.sc_dev
;
1781 for (i
= 0; i
< hdl
->device
.max_enabled_vfs
; i
++) {
1782 if (!dev
->vf_dev
[i
] || (dev
->vf_dev
[i
]->vf_id
== vf_id
))
1790 * i40iw_virtchnl_receive - receive a message through the virtual channel
1791 * @ldev: lan device information
1792 * @client: client interface instance
1793 * @vf_id: virtual function id associated with the message
1794 * @msg: message buffer pointer
1795 * @len: length of the message
1797 * Invoke virtual channel receive operation for the given msg
1798 * Return 0 if successful, otherwise return error
1800 static int i40iw_virtchnl_receive(struct i40e_info
*ldev
,
1801 struct i40e_client
*client
,
1806 struct i40iw_handler
*hdl
;
1807 struct i40iw_sc_dev
*dev
;
1808 struct i40iw_device
*iwdev
;
1809 int ret_code
= I40IW_NOT_SUPPORTED
;
1812 return I40IW_ERR_PARAM
;
1814 hdl
= i40iw_find_i40e_handler(ldev
);
1816 return I40IW_ERR_PARAM
;
1818 dev
= &hdl
->device
.sc_dev
;
1819 iwdev
= dev
->back_dev
;
1821 if (dev
->vchnl_if
.vchnl_recv
) {
1822 ret_code
= dev
->vchnl_if
.vchnl_recv(dev
, vf_id
, msg
, len
);
1824 atomic_dec(&iwdev
->vchnl_msgs
);
1825 wake_up(&iwdev
->vchnl_waitq
);
1832 * i40iw_vf_clear_to_send - wait to send virtual channel message
1833 * @dev: iwarp device *
1834 * Wait for until virtual channel is clear
1835 * before sending the next message
1837 * Returns false if error
1838 * Returns true if clear to send
1840 bool i40iw_vf_clear_to_send(struct i40iw_sc_dev
*dev
)
1842 struct i40iw_device
*iwdev
;
1845 iwdev
= dev
->back_dev
;
1847 if (!wq_has_sleeper(&dev
->vf_reqs
) &&
1848 (atomic_read(&iwdev
->vchnl_msgs
) == 0))
1849 return true; /* virtual channel is clear */
1852 add_wait_queue_exclusive(&dev
->vf_reqs
, &wait
);
1854 if (!wait_event_timeout(dev
->vf_reqs
,
1855 (atomic_read(&iwdev
->vchnl_msgs
) == 0),
1856 I40IW_VCHNL_EVENT_TIMEOUT
))
1857 dev
->vchnl_up
= false;
1859 remove_wait_queue(&dev
->vf_reqs
, &wait
);
1861 return dev
->vchnl_up
;
1865 * i40iw_virtchnl_send - send a message through the virtual channel
1866 * @dev: iwarp device
1867 * @vf_id: virtual function id associated with the message
1868 * @msg: virtual channel message buffer pointer
1869 * @len: length of the message
1871 * Invoke virtual channel send operation for the given msg
1872 * Return 0 if successful, otherwise return error
1874 static enum i40iw_status_code
i40iw_virtchnl_send(struct i40iw_sc_dev
*dev
,
1879 struct i40iw_device
*iwdev
;
1880 struct i40e_info
*ldev
;
1882 if (!dev
|| !dev
->back_dev
)
1883 return I40IW_ERR_BAD_PTR
;
1885 iwdev
= dev
->back_dev
;
1888 if (ldev
&& ldev
->ops
&& ldev
->ops
->virtchnl_send
)
1889 return ldev
->ops
->virtchnl_send(ldev
, &i40iw_client
, vf_id
, msg
, len
);
1890 return I40IW_ERR_BAD_PTR
;
1893 /* client interface functions */
1894 static const struct i40e_client_ops i40e_ops
= {
1896 .close
= i40iw_close
,
1897 .l2_param_change
= i40iw_l2param_change
,
1898 .virtchnl_receive
= i40iw_virtchnl_receive
,
1899 .vf_reset
= i40iw_vf_reset
,
1900 .vf_enable
= i40iw_vf_enable
,
1901 .vf_capable
= i40iw_vf_capable
1905 * i40iw_init_module - driver initialization function
1907 * First function to call when the driver is loaded
1908 * Register the driver as i40e client and port mapper client
1910 static int __init
i40iw_init_module(void)
1914 memset(&i40iw_client
, 0, sizeof(i40iw_client
));
1915 i40iw_client
.version
.major
= CLIENT_IW_INTERFACE_VERSION_MAJOR
;
1916 i40iw_client
.version
.minor
= CLIENT_IW_INTERFACE_VERSION_MINOR
;
1917 i40iw_client
.version
.build
= CLIENT_IW_INTERFACE_VERSION_BUILD
;
1918 i40iw_client
.ops
= &i40e_ops
;
1919 memcpy(i40iw_client
.name
, i40iw_client_name
, I40E_CLIENT_STR_LENGTH
);
1920 i40iw_client
.type
= I40E_CLIENT_IWARP
;
1921 spin_lock_init(&i40iw_handler_lock
);
1922 ret
= i40e_register_client(&i40iw_client
);
1927 * i40iw_exit_module - driver exit clean up function
1929 * The function is called just before the driver is unloaded
1930 * Unregister the driver as i40e client and port mapper client
1932 static void __exit
i40iw_exit_module(void)
1934 i40e_unregister_client(&i40iw_client
);
1937 module_init(i40iw_init_module
);
1938 module_exit(i40iw_exit_module
);