Commit | Line | Data |
---|---|---|
f931551b RC |
1 | /* |
2 | * Copyright (c) 2006, 2007, 2009 QLogic Corporation. All rights reserved. | |
3 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. | |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | */ | |
33 | ||
34 | #include "qib.h" | |
35 | ||
36 | /** | |
37 | * qib_alloc_lkey - allocate an lkey | |
f931551b | 38 | * @mr: memory region that this lkey protects |
6a82649f MM |
39 | * @dma_region: 0->normal key, 1->restricted DMA key |
40 | * | |
41 | * Returns 0 if successful, otherwise returns -errno. | |
42 | * | |
8aac4cc3 | 43 | * Increments mr reference count as required. |
6a82649f MM |
44 | * |
45 | * Sets the lkey field mr for non-dma regions. | |
f931551b | 46 | * |
f931551b RC |
47 | */ |
48 | ||
7c2e11fe | 49 | int qib_alloc_lkey(struct rvt_mregion *mr, int dma_region) |
f931551b RC |
50 | { |
51 | unsigned long flags; | |
52 | u32 r; | |
53 | u32 n; | |
6a82649f MM |
54 | int ret = 0; |
55 | struct qib_ibdev *dev = to_idev(mr->pd->device); | |
7c2e11fe | 56 | struct rvt_lkey_table *rkt = &dev->lk_table; |
f931551b RC |
57 | |
58 | spin_lock_irqsave(&rkt->lock, flags); | |
59 | ||
6a82649f MM |
60 | /* special case for dma_mr lkey == 0 */ |
61 | if (dma_region) { | |
7c2e11fe | 62 | struct rvt_mregion *tmr; |
8aac4cc3 | 63 | |
f3bdf344 | 64 | tmr = rcu_access_pointer(dev->dma_mr); |
8aac4cc3 | 65 | if (!tmr) { |
6a82649f | 66 | qib_get_mr(mr); |
8aac4cc3 | 67 | rcu_assign_pointer(dev->dma_mr, mr); |
6a82649f MM |
68 | mr->lkey_published = 1; |
69 | } | |
70 | goto success; | |
71 | } | |
72 | ||
f931551b RC |
73 | /* Find the next available LKEY */ |
74 | r = rkt->next; | |
75 | n = r; | |
76 | for (;;) { | |
77 | if (rkt->table[r] == NULL) | |
78 | break; | |
79 | r = (r + 1) & (rkt->max - 1); | |
6a82649f | 80 | if (r == n) |
f931551b | 81 | goto bail; |
f931551b RC |
82 | } |
83 | rkt->next = (r + 1) & (rkt->max - 1); | |
84 | /* | |
85 | * Make sure lkey is never zero which is reserved to indicate an | |
86 | * unrestricted LKEY. | |
87 | */ | |
88 | rkt->gen++; | |
d6f1c17e MM |
89 | /* |
90 | * bits are capped in qib_verbs.c to insure enough bits | |
91 | * for generation number | |
92 | */ | |
7c2e11fe DD |
93 | mr->lkey = (r << (32 - ib_rvt_lkey_table_size)) | |
94 | ((((1 << (24 - ib_rvt_lkey_table_size)) - 1) & rkt->gen) | |
f931551b RC |
95 | << 8); |
96 | if (mr->lkey == 0) { | |
97 | mr->lkey |= 1 << 8; | |
98 | rkt->gen++; | |
99 | } | |
6a82649f | 100 | qib_get_mr(mr); |
8aac4cc3 | 101 | rcu_assign_pointer(rkt->table[r], mr); |
6a82649f MM |
102 | mr->lkey_published = 1; |
103 | success: | |
f931551b | 104 | spin_unlock_irqrestore(&rkt->lock, flags); |
6a82649f | 105 | out: |
f931551b | 106 | return ret; |
6a82649f MM |
107 | bail: |
108 | spin_unlock_irqrestore(&rkt->lock, flags); | |
109 | ret = -ENOMEM; | |
110 | goto out; | |
f931551b RC |
111 | } |
112 | ||
113 | /** | |
114 | * qib_free_lkey - free an lkey | |
6a82649f | 115 | * @mr: mr to free from tables |
f931551b | 116 | */ |
7c2e11fe | 117 | void qib_free_lkey(struct rvt_mregion *mr) |
f931551b RC |
118 | { |
119 | unsigned long flags; | |
120 | u32 lkey = mr->lkey; | |
121 | u32 r; | |
6a82649f | 122 | struct qib_ibdev *dev = to_idev(mr->pd->device); |
7c2e11fe | 123 | struct rvt_lkey_table *rkt = &dev->lk_table; |
6a82649f MM |
124 | |
125 | spin_lock_irqsave(&rkt->lock, flags); | |
126 | if (!mr->lkey_published) | |
127 | goto out; | |
8aac4cc3 | 128 | if (lkey == 0) |
590c3fec | 129 | RCU_INIT_POINTER(dev->dma_mr, NULL); |
8aac4cc3 | 130 | else { |
7c2e11fe | 131 | r = lkey >> (32 - ib_rvt_lkey_table_size); |
590c3fec | 132 | RCU_INIT_POINTER(rkt->table[r], NULL); |
f931551b | 133 | } |
8aac4cc3 MM |
134 | qib_put_mr(mr); |
135 | mr->lkey_published = 0; | |
6a82649f | 136 | out: |
8aac4cc3 | 137 | spin_unlock_irqrestore(&rkt->lock, flags); |
f931551b RC |
138 | } |
139 | ||
f931551b RC |
140 | /** |
141 | * qib_rkey_ok - check the IB virtual address, length, and RKEY | |
8aac4cc3 MM |
142 | * @qp: qp for validation |
143 | * @sge: SGE state | |
f931551b RC |
144 | * @len: length of data |
145 | * @vaddr: virtual address to place data | |
146 | * @rkey: rkey to check | |
147 | * @acc: access flags | |
148 | * | |
149 | * Return 1 if successful, otherwise 0. | |
8aac4cc3 MM |
150 | * |
151 | * increments the reference count upon success | |
f931551b | 152 | */ |
7c2e11fe | 153 | int qib_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge, |
f931551b RC |
154 | u32 len, u64 vaddr, u32 rkey, int acc) |
155 | { | |
7c2e11fe DD |
156 | struct rvt_lkey_table *rkt = &to_idev(qp->ibqp.device)->lk_table; |
157 | struct rvt_mregion *mr; | |
f931551b RC |
158 | unsigned n, m; |
159 | size_t off; | |
f931551b RC |
160 | |
161 | /* | |
162 | * We use RKEY == zero for kernel virtual addresses | |
163 | * (see qib_get_dma_mr and qib_dma.c). | |
164 | */ | |
8aac4cc3 | 165 | rcu_read_lock(); |
f931551b | 166 | if (rkey == 0) { |
f44728d6 | 167 | struct rvt_pd *pd = ibpd_to_rvtpd(qp->ibqp.pd); |
f931551b RC |
168 | struct qib_ibdev *dev = to_idev(pd->ibpd.device); |
169 | ||
170 | if (pd->user) | |
171 | goto bail; | |
8aac4cc3 MM |
172 | mr = rcu_dereference(dev->dma_mr); |
173 | if (!mr) | |
174 | goto bail; | |
175 | if (unlikely(!atomic_inc_not_zero(&mr->refcount))) | |
f931551b | 176 | goto bail; |
8aac4cc3 | 177 | rcu_read_unlock(); |
4db62d47 | 178 | |
8aac4cc3 | 179 | sge->mr = mr; |
f931551b RC |
180 | sge->vaddr = (void *) vaddr; |
181 | sge->length = len; | |
182 | sge->sge_length = len; | |
183 | sge->m = 0; | |
184 | sge->n = 0; | |
185 | goto ok; | |
186 | } | |
187 | ||
8aac4cc3 | 188 | mr = rcu_dereference( |
7c2e11fe | 189 | rkt->table[(rkey >> (32 - ib_rvt_lkey_table_size))]); |
8aac4cc3 | 190 | if (unlikely(!mr || mr->lkey != rkey || qp->ibqp.pd != mr->pd)) |
f931551b RC |
191 | goto bail; |
192 | ||
193 | off = vaddr - mr->iova; | |
194 | if (unlikely(vaddr < mr->iova || off + len > mr->length || | |
195 | (mr->access_flags & acc) == 0)) | |
4db62d47 | 196 | goto bail; |
8aac4cc3 MM |
197 | if (unlikely(!atomic_inc_not_zero(&mr->refcount))) |
198 | goto bail; | |
199 | rcu_read_unlock(); | |
f931551b RC |
200 | |
201 | off += mr->offset; | |
2a600f14 MM |
202 | if (mr->page_shift) { |
203 | /* | |
204 | page sizes are uniform power of 2 so no loop is necessary | |
205 | entries_spanned_by_off is the number of times the loop below | |
206 | would have executed. | |
207 | */ | |
208 | size_t entries_spanned_by_off; | |
209 | ||
210 | entries_spanned_by_off = off >> mr->page_shift; | |
211 | off -= (entries_spanned_by_off << mr->page_shift); | |
7c2e11fe DD |
212 | m = entries_spanned_by_off / RVT_SEGSZ; |
213 | n = entries_spanned_by_off % RVT_SEGSZ; | |
2a600f14 MM |
214 | } else { |
215 | m = 0; | |
216 | n = 0; | |
217 | while (off >= mr->map[m]->segs[n].length) { | |
218 | off -= mr->map[m]->segs[n].length; | |
219 | n++; | |
7c2e11fe | 220 | if (n >= RVT_SEGSZ) { |
2a600f14 MM |
221 | m++; |
222 | n = 0; | |
223 | } | |
f931551b RC |
224 | } |
225 | } | |
f931551b RC |
226 | sge->mr = mr; |
227 | sge->vaddr = mr->map[m]->segs[n].vaddr + off; | |
228 | sge->length = mr->map[m]->segs[n].length - off; | |
229 | sge->sge_length = len; | |
230 | sge->m = m; | |
231 | sge->n = n; | |
232 | ok: | |
4db62d47 | 233 | return 1; |
f931551b | 234 | bail: |
8aac4cc3 | 235 | rcu_read_unlock(); |
4db62d47 | 236 | return 0; |
f931551b RC |
237 | } |
238 |