3 * This file is provided under a dual BSD/GPLv2 license. When using or
4 * redistributing this file, you may do so under either license.
8 * Copyright(c) 2015 Intel Corporation.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * General Public License for more details.
21 * Copyright(c) 2015 Intel Corporation.
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
27 * - Redistributions of source code must retain the above copyright
28 * notice, this list of conditions and the following disclaimer.
29 * - Redistributions in binary form must reproduce the above copyright
30 * notice, this list of conditions and the following disclaimer in
31 * the documentation and/or other materials provided with the
33 * - Neither the name of Intel Corporation nor the names of its
34 * contributors may be used to endorse or promote products derived
35 * from this software without specific prior written permission.
37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
54 * hfi1_alloc_lkey - allocate an lkey
55 * @mr: memory region that this lkey protects
56 * @dma_region: 0->normal key, 1->restricted DMA key
58 * Returns 0 if successful, otherwise returns -errno.
60 * Increments mr reference count as required.
62 * Sets the lkey field mr for non-dma regions.
66 int hfi1_alloc_lkey(struct hfi1_mregion
*mr
, int dma_region
)
72 struct hfi1_ibdev
*dev
= to_idev(mr
->pd
->device
);
73 struct hfi1_lkey_table
*rkt
= &dev
->lk_table
;
76 spin_lock_irqsave(&rkt
->lock
, flags
);
78 /* special case for dma_mr lkey == 0 */
80 struct hfi1_mregion
*tmr
;
82 tmr
= rcu_access_pointer(dev
->dma_mr
);
84 rcu_assign_pointer(dev
->dma_mr
, mr
);
85 mr
->lkey_published
= 1;
92 /* Find the next available LKEY */
96 if (!rcu_access_pointer(rkt
->table
[r
]))
98 r
= (r
+ 1) & (rkt
->max
- 1);
102 rkt
->next
= (r
+ 1) & (rkt
->max
- 1);
104 * Make sure lkey is never zero which is reserved to indicate an
109 * bits are capped in verbs.c to ensure enough bits for
112 mr
->lkey
= (r
<< (32 - hfi1_lkey_table_size
)) |
113 ((((1 << (24 - hfi1_lkey_table_size
)) - 1) & rkt
->gen
)
119 rcu_assign_pointer(rkt
->table
[r
], mr
);
120 mr
->lkey_published
= 1;
122 spin_unlock_irqrestore(&rkt
->lock
, flags
);
127 spin_unlock_irqrestore(&rkt
->lock
, flags
);
133 * hfi1_free_lkey - free an lkey
134 * @mr: mr to free from tables
136 void hfi1_free_lkey(struct hfi1_mregion
*mr
)
141 struct hfi1_ibdev
*dev
= to_idev(mr
->pd
->device
);
142 struct hfi1_lkey_table
*rkt
= &dev
->lk_table
;
145 spin_lock_irqsave(&rkt
->lock
, flags
);
146 if (!mr
->lkey_published
)
149 RCU_INIT_POINTER(dev
->dma_mr
, NULL
);
151 r
= lkey
>> (32 - hfi1_lkey_table_size
);
152 RCU_INIT_POINTER(rkt
->table
[r
], NULL
);
154 mr
->lkey_published
= 0;
157 spin_unlock_irqrestore(&rkt
->lock
, flags
);
165 * hfi1_lkey_ok - check IB SGE for validity and initialize
166 * @rkt: table containing lkey to check SGE against
167 * @pd: protection domain
168 * @isge: outgoing internal SGE
172 * Return 1 if valid and successful, otherwise returns 0.
174 * increments the reference count upon success
176 * Check the IB SGE for validity and initialize our internal version
179 int hfi1_lkey_ok(struct hfi1_lkey_table
*rkt
, struct hfi1_pd
*pd
,
180 struct hfi1_sge
*isge
, struct ib_sge
*sge
, int acc
)
182 struct hfi1_mregion
*mr
;
187 * We use LKEY == zero for kernel virtual addresses
188 * (see hfi1_get_dma_mr and dma.c).
191 if (sge
->lkey
== 0) {
192 struct hfi1_ibdev
*dev
= to_idev(pd
->ibpd
.device
);
196 mr
= rcu_dereference(dev
->dma_mr
);
199 atomic_inc(&mr
->refcount
);
203 isge
->vaddr
= (void *) sge
->addr
;
204 isge
->length
= sge
->length
;
205 isge
->sge_length
= sge
->length
;
210 mr
= rcu_dereference(
211 rkt
->table
[(sge
->lkey
>> (32 - hfi1_lkey_table_size
))]);
212 if (unlikely(!mr
|| mr
->lkey
!= sge
->lkey
|| mr
->pd
!= &pd
->ibpd
))
215 off
= sge
->addr
- mr
->user_base
;
216 if (unlikely(sge
->addr
< mr
->user_base
||
217 off
+ sge
->length
> mr
->length
||
218 (mr
->access_flags
& acc
) != acc
))
220 atomic_inc(&mr
->refcount
);
224 if (mr
->page_shift
) {
226 page sizes are uniform power of 2 so no loop is necessary
227 entries_spanned_by_off is the number of times the loop below
230 size_t entries_spanned_by_off
;
232 entries_spanned_by_off
= off
>> mr
->page_shift
;
233 off
-= (entries_spanned_by_off
<< mr
->page_shift
);
234 m
= entries_spanned_by_off
/ HFI1_SEGSZ
;
235 n
= entries_spanned_by_off
% HFI1_SEGSZ
;
239 while (off
>= mr
->map
[m
]->segs
[n
].length
) {
240 off
-= mr
->map
[m
]->segs
[n
].length
;
242 if (n
>= HFI1_SEGSZ
) {
249 isge
->vaddr
= mr
->map
[m
]->segs
[n
].vaddr
+ off
;
250 isge
->length
= mr
->map
[m
]->segs
[n
].length
- off
;
251 isge
->sge_length
= sge
->length
;
262 * hfi1_rkey_ok - check the IB virtual address, length, and RKEY
263 * @qp: qp for validation
265 * @len: length of data
266 * @vaddr: virtual address to place data
267 * @rkey: rkey to check
270 * Return 1 if successful, otherwise 0.
272 * increments the reference count upon success
274 int hfi1_rkey_ok(struct hfi1_qp
*qp
, struct hfi1_sge
*sge
,
275 u32 len
, u64 vaddr
, u32 rkey
, int acc
)
277 struct hfi1_lkey_table
*rkt
= &to_idev(qp
->ibqp
.device
)->lk_table
;
278 struct hfi1_mregion
*mr
;
283 * We use RKEY == zero for kernel virtual addresses
284 * (see hfi1_get_dma_mr and dma.c).
288 struct hfi1_pd
*pd
= to_ipd(qp
->ibqp
.pd
);
289 struct hfi1_ibdev
*dev
= to_idev(pd
->ibpd
.device
);
293 mr
= rcu_dereference(dev
->dma_mr
);
296 atomic_inc(&mr
->refcount
);
300 sge
->vaddr
= (void *) vaddr
;
302 sge
->sge_length
= len
;
308 mr
= rcu_dereference(
309 rkt
->table
[(rkey
>> (32 - hfi1_lkey_table_size
))]);
310 if (unlikely(!mr
|| mr
->lkey
!= rkey
|| qp
->ibqp
.pd
!= mr
->pd
))
313 off
= vaddr
- mr
->iova
;
314 if (unlikely(vaddr
< mr
->iova
|| off
+ len
> mr
->length
||
315 (mr
->access_flags
& acc
) == 0))
317 atomic_inc(&mr
->refcount
);
321 if (mr
->page_shift
) {
323 page sizes are uniform power of 2 so no loop is necessary
324 entries_spanned_by_off is the number of times the loop below
327 size_t entries_spanned_by_off
;
329 entries_spanned_by_off
= off
>> mr
->page_shift
;
330 off
-= (entries_spanned_by_off
<< mr
->page_shift
);
331 m
= entries_spanned_by_off
/ HFI1_SEGSZ
;
332 n
= entries_spanned_by_off
% HFI1_SEGSZ
;
336 while (off
>= mr
->map
[m
]->segs
[n
].length
) {
337 off
-= mr
->map
[m
]->segs
[n
].length
;
339 if (n
>= HFI1_SEGSZ
) {
346 sge
->vaddr
= mr
->map
[m
]->segs
[n
].vaddr
+ off
;
347 sge
->length
= mr
->map
[m
]->segs
[n
].length
- off
;
348 sge
->sge_length
= len
;
This page took 0.040333 seconds and 5 git commands to generate.