Commit | Line | Data |
---|---|---|
a9dcad5e HD |
1 | /* |
2 | * omap iommu: tlb and pagetable primitives | |
3 | * | |
c127c7dc | 4 | * Copyright (C) 2008-2010 Nokia Corporation |
a9dcad5e HD |
5 | * |
6 | * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>, | |
7 | * Paul Mundt and Toshihiro Kobayashi | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License version 2 as | |
11 | * published by the Free Software Foundation. | |
12 | */ | |
13 | ||
14 | #include <linux/err.h> | |
5a0e3ad6 | 15 | #include <linux/slab.h> |
a9dcad5e HD |
16 | #include <linux/interrupt.h> |
17 | #include <linux/ioport.h> | |
a9dcad5e | 18 | #include <linux/platform_device.h> |
f626b52d | 19 | #include <linux/iommu.h> |
c8d35c84 | 20 | #include <linux/omap-iommu.h> |
f626b52d OBC |
21 | #include <linux/mutex.h> |
22 | #include <linux/spinlock.h> | |
ed1c7de2 | 23 | #include <linux/io.h> |
ebf7cda0 | 24 | #include <linux/pm_runtime.h> |
3c92748d FV |
25 | #include <linux/of.h> |
26 | #include <linux/of_iommu.h> | |
27 | #include <linux/of_irq.h> | |
7d682774 | 28 | #include <linux/of_platform.h> |
a9dcad5e HD |
29 | |
30 | #include <asm/cacheflush.h> | |
31 | ||
2ab7c848 | 32 | #include <linux/platform_data/iommu-omap.h> |
a9dcad5e | 33 | |
2f7702af | 34 | #include "omap-iopgtable.h" |
ed1c7de2 | 35 | #include "omap-iommu.h" |
a9dcad5e | 36 | |
5acc97db SA |
37 | #define to_iommu(dev) \ |
38 | ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev))) | |
39 | ||
66bc8cf3 OBC |
40 | /* bitmap of the page sizes currently supported */ |
41 | #define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) | |
42 | ||
f626b52d OBC |
43 | /** |
44 | * struct omap_iommu_domain - omap iommu domain | |
45 | * @pgtable: the page table | |
46 | * @iommu_dev: an omap iommu device attached to this domain. only a single | |
47 | * iommu device can be attached for now. | |
803b5277 | 48 | * @dev: Device using this domain. |
f626b52d OBC |
49 | * @lock: domain lock, should be taken when attaching/detaching |
50 | */ | |
51 | struct omap_iommu_domain { | |
52 | u32 *pgtable; | |
6c32df43 | 53 | struct omap_iommu *iommu_dev; |
803b5277 | 54 | struct device *dev; |
f626b52d | 55 | spinlock_t lock; |
8cf851e0 | 56 | struct iommu_domain domain; |
f626b52d OBC |
57 | }; |
58 | ||
7bd9e25f IY |
59 | #define MMU_LOCK_BASE_SHIFT 10 |
60 | #define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT) | |
61 | #define MMU_LOCK_BASE(x) \ | |
62 | ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT) | |
63 | ||
64 | #define MMU_LOCK_VICT_SHIFT 4 | |
65 | #define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT) | |
66 | #define MMU_LOCK_VICT(x) \ | |
67 | ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT) | |
68 | ||
a9dcad5e HD |
69 | static struct platform_driver omap_iommu_driver; |
70 | static struct kmem_cache *iopte_cachep; | |
71 | ||
8cf851e0 JR |
72 | /** |
73 | * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain | |
74 | * @dom: generic iommu domain handle | |
75 | **/ | |
76 | static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom) | |
77 | { | |
78 | return container_of(dom, struct omap_iommu_domain, domain); | |
79 | } | |
80 | ||
a9dcad5e | 81 | /** |
6c32df43 | 82 | * omap_iommu_save_ctx - Save registers for pm off-mode support |
fabdbca8 | 83 | * @dev: client device |
a9dcad5e | 84 | **/ |
fabdbca8 | 85 | void omap_iommu_save_ctx(struct device *dev) |
a9dcad5e | 86 | { |
fabdbca8 | 87 | struct omap_iommu *obj = dev_to_omap_iommu(dev); |
bd4396f0 SA |
88 | u32 *p = obj->ctx; |
89 | int i; | |
fabdbca8 | 90 | |
bd4396f0 SA |
91 | for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { |
92 | p[i] = iommu_read_reg(obj, i * sizeof(u32)); | |
93 | dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]); | |
94 | } | |
a9dcad5e | 95 | } |
6c32df43 | 96 | EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); |
a9dcad5e HD |
97 | |
98 | /** | |
6c32df43 | 99 | * omap_iommu_restore_ctx - Restore registers for pm off-mode support |
fabdbca8 | 100 | * @dev: client device |
a9dcad5e | 101 | **/ |
fabdbca8 | 102 | void omap_iommu_restore_ctx(struct device *dev) |
a9dcad5e | 103 | { |
fabdbca8 | 104 | struct omap_iommu *obj = dev_to_omap_iommu(dev); |
bd4396f0 SA |
105 | u32 *p = obj->ctx; |
106 | int i; | |
fabdbca8 | 107 | |
bd4396f0 SA |
108 | for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { |
109 | iommu_write_reg(obj, p[i], i * sizeof(u32)); | |
110 | dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]); | |
111 | } | |
a9dcad5e | 112 | } |
6c32df43 | 113 | EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx); |
a9dcad5e | 114 | |
bd4396f0 SA |
115 | static void __iommu_set_twl(struct omap_iommu *obj, bool on) |
116 | { | |
117 | u32 l = iommu_read_reg(obj, MMU_CNTL); | |
118 | ||
119 | if (on) | |
120 | iommu_write_reg(obj, MMU_IRQ_TWL_MASK, MMU_IRQENABLE); | |
121 | else | |
122 | iommu_write_reg(obj, MMU_IRQ_TLB_MISS_MASK, MMU_IRQENABLE); | |
123 | ||
124 | l &= ~MMU_CNTL_MASK; | |
125 | if (on) | |
126 | l |= (MMU_CNTL_MMU_EN | MMU_CNTL_TWL_EN); | |
127 | else | |
128 | l |= (MMU_CNTL_MMU_EN); | |
129 | ||
130 | iommu_write_reg(obj, l, MMU_CNTL); | |
131 | } | |
132 | ||
133 | static int omap2_iommu_enable(struct omap_iommu *obj) | |
134 | { | |
135 | u32 l, pa; | |
136 | ||
137 | if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd, SZ_16K)) | |
138 | return -EINVAL; | |
139 | ||
140 | pa = virt_to_phys(obj->iopgd); | |
141 | if (!IS_ALIGNED(pa, SZ_16K)) | |
142 | return -EINVAL; | |
143 | ||
144 | l = iommu_read_reg(obj, MMU_REVISION); | |
145 | dev_info(obj->dev, "%s: version %d.%d\n", obj->name, | |
146 | (l >> 4) & 0xf, l & 0xf); | |
147 | ||
148 | iommu_write_reg(obj, pa, MMU_TTB); | |
149 | ||
150 | if (obj->has_bus_err_back) | |
151 | iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG); | |
152 | ||
153 | __iommu_set_twl(obj, true); | |
154 | ||
155 | return 0; | |
156 | } | |
157 | ||
158 | static void omap2_iommu_disable(struct omap_iommu *obj) | |
159 | { | |
160 | u32 l = iommu_read_reg(obj, MMU_CNTL); | |
161 | ||
162 | l &= ~MMU_CNTL_MASK; | |
163 | iommu_write_reg(obj, l, MMU_CNTL); | |
164 | ||
165 | dev_dbg(obj->dev, "%s is shutting down\n", obj->name); | |
166 | } | |
167 | ||
6c32df43 | 168 | static int iommu_enable(struct omap_iommu *obj) |
a9dcad5e HD |
169 | { |
170 | int err; | |
72b15b6a | 171 | struct platform_device *pdev = to_platform_device(obj->dev); |
99cb9aee | 172 | struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev); |
a9dcad5e | 173 | |
90e569c4 | 174 | if (pdata && pdata->deassert_reset) { |
72b15b6a ORL |
175 | err = pdata->deassert_reset(pdev, pdata->reset_name); |
176 | if (err) { | |
177 | dev_err(obj->dev, "deassert_reset failed: %d\n", err); | |
178 | return err; | |
179 | } | |
180 | } | |
181 | ||
ebf7cda0 | 182 | pm_runtime_get_sync(obj->dev); |
a9dcad5e | 183 | |
bd4396f0 | 184 | err = omap2_iommu_enable(obj); |
a9dcad5e | 185 | |
a9dcad5e HD |
186 | return err; |
187 | } | |
188 | ||
6c32df43 | 189 | static void iommu_disable(struct omap_iommu *obj) |
a9dcad5e | 190 | { |
72b15b6a | 191 | struct platform_device *pdev = to_platform_device(obj->dev); |
99cb9aee | 192 | struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev); |
72b15b6a | 193 | |
bd4396f0 | 194 | omap2_iommu_disable(obj); |
a9dcad5e | 195 | |
ebf7cda0 | 196 | pm_runtime_put_sync(obj->dev); |
72b15b6a | 197 | |
90e569c4 | 198 | if (pdata && pdata->assert_reset) |
72b15b6a | 199 | pdata->assert_reset(pdev, pdata->reset_name); |
a9dcad5e HD |
200 | } |
201 | ||
202 | /* | |
203 | * TLB operations | |
204 | */ | |
e1f23813 | 205 | static u32 iotlb_cr_to_virt(struct cr_regs *cr) |
a9dcad5e | 206 | { |
bd4396f0 SA |
207 | u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK; |
208 | u32 mask = get_cam_va_mask(cr->cam & page_size); | |
209 | ||
210 | return cr->cam & mask; | |
a9dcad5e | 211 | } |
a9dcad5e HD |
212 | |
213 | static u32 get_iopte_attr(struct iotlb_entry *e) | |
214 | { | |
bd4396f0 SA |
215 | u32 attr; |
216 | ||
217 | attr = e->mixed << 5; | |
218 | attr |= e->endian; | |
219 | attr |= e->elsz >> 3; | |
220 | attr <<= (((e->pgsz == MMU_CAM_PGSZ_4K) || | |
221 | (e->pgsz == MMU_CAM_PGSZ_64K)) ? 0 : 6); | |
222 | return attr; | |
a9dcad5e HD |
223 | } |
224 | ||
6c32df43 | 225 | static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da) |
a9dcad5e | 226 | { |
bd4396f0 SA |
227 | u32 status, fault_addr; |
228 | ||
229 | status = iommu_read_reg(obj, MMU_IRQSTATUS); | |
230 | status &= MMU_IRQ_MASK; | |
231 | if (!status) { | |
232 | *da = 0; | |
233 | return 0; | |
234 | } | |
235 | ||
236 | fault_addr = iommu_read_reg(obj, MMU_FAULT_AD); | |
237 | *da = fault_addr; | |
238 | ||
239 | iommu_write_reg(obj, status, MMU_IRQSTATUS); | |
240 | ||
241 | return status; | |
a9dcad5e HD |
242 | } |
243 | ||
69c2c196 | 244 | void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) |
a9dcad5e HD |
245 | { |
246 | u32 val; | |
247 | ||
248 | val = iommu_read_reg(obj, MMU_LOCK); | |
249 | ||
250 | l->base = MMU_LOCK_BASE(val); | |
251 | l->vict = MMU_LOCK_VICT(val); | |
a9dcad5e HD |
252 | } |
253 | ||
69c2c196 | 254 | void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) |
a9dcad5e HD |
255 | { |
256 | u32 val; | |
257 | ||
a9dcad5e HD |
258 | val = (l->base << MMU_LOCK_BASE_SHIFT); |
259 | val |= (l->vict << MMU_LOCK_VICT_SHIFT); | |
260 | ||
261 | iommu_write_reg(obj, val, MMU_LOCK); | |
262 | } | |
263 | ||
6c32df43 | 264 | static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr) |
a9dcad5e | 265 | { |
bd4396f0 SA |
266 | cr->cam = iommu_read_reg(obj, MMU_READ_CAM); |
267 | cr->ram = iommu_read_reg(obj, MMU_READ_RAM); | |
a9dcad5e HD |
268 | } |
269 | ||
6c32df43 | 270 | static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr) |
a9dcad5e | 271 | { |
bd4396f0 SA |
272 | iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM); |
273 | iommu_write_reg(obj, cr->ram, MMU_RAM); | |
a9dcad5e HD |
274 | |
275 | iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); | |
276 | iommu_write_reg(obj, 1, MMU_LD_TLB); | |
277 | } | |
278 | ||
37c2836c | 279 | /* only used in iotlb iteration for-loop */ |
69c2c196 | 280 | struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) |
37c2836c HD |
281 | { |
282 | struct cr_regs cr; | |
283 | struct iotlb_lock l; | |
284 | ||
285 | iotlb_lock_get(obj, &l); | |
286 | l.vict = n; | |
287 | iotlb_lock_set(obj, &l); | |
288 | iotlb_read_cr(obj, &cr); | |
289 | ||
290 | return cr; | |
291 | } | |
292 | ||
bd4396f0 SA |
293 | #ifdef PREFETCH_IOTLB |
294 | static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj, | |
295 | struct iotlb_entry *e) | |
296 | { | |
297 | struct cr_regs *cr; | |
298 | ||
299 | if (!e) | |
300 | return NULL; | |
301 | ||
302 | if (e->da & ~(get_cam_va_mask(e->pgsz))) { | |
303 | dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__, | |
304 | e->da); | |
305 | return ERR_PTR(-EINVAL); | |
306 | } | |
307 | ||
308 | cr = kmalloc(sizeof(*cr), GFP_KERNEL); | |
309 | if (!cr) | |
310 | return ERR_PTR(-ENOMEM); | |
311 | ||
312 | cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid; | |
313 | cr->ram = e->pa | e->endian | e->elsz | e->mixed; | |
314 | ||
315 | return cr; | |
316 | } | |
317 | ||
a9dcad5e HD |
318 | /** |
319 | * load_iotlb_entry - Set an iommu tlb entry | |
320 | * @obj: target iommu | |
321 | * @e: an iommu tlb entry info | |
322 | **/ | |
6c32df43 | 323 | static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
a9dcad5e | 324 | { |
a9dcad5e HD |
325 | int err = 0; |
326 | struct iotlb_lock l; | |
327 | struct cr_regs *cr; | |
328 | ||
329 | if (!obj || !obj->nr_tlb_entries || !e) | |
330 | return -EINVAL; | |
331 | ||
ebf7cda0 | 332 | pm_runtime_get_sync(obj->dev); |
a9dcad5e | 333 | |
be6d8026 KH |
334 | iotlb_lock_get(obj, &l); |
335 | if (l.base == obj->nr_tlb_entries) { | |
336 | dev_warn(obj->dev, "%s: preserve entries full\n", __func__); | |
a9dcad5e HD |
337 | err = -EBUSY; |
338 | goto out; | |
339 | } | |
be6d8026 | 340 | if (!e->prsvd) { |
37c2836c HD |
341 | int i; |
342 | struct cr_regs tmp; | |
be6d8026 | 343 | |
37c2836c | 344 | for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp) |
be6d8026 KH |
345 | if (!iotlb_cr_valid(&tmp)) |
346 | break; | |
37c2836c | 347 | |
be6d8026 KH |
348 | if (i == obj->nr_tlb_entries) { |
349 | dev_dbg(obj->dev, "%s: full: no entry\n", __func__); | |
350 | err = -EBUSY; | |
351 | goto out; | |
352 | } | |
37c2836c HD |
353 | |
354 | iotlb_lock_get(obj, &l); | |
be6d8026 KH |
355 | } else { |
356 | l.vict = l.base; | |
357 | iotlb_lock_set(obj, &l); | |
358 | } | |
a9dcad5e HD |
359 | |
360 | cr = iotlb_alloc_cr(obj, e); | |
361 | if (IS_ERR(cr)) { | |
ebf7cda0 | 362 | pm_runtime_put_sync(obj->dev); |
a9dcad5e HD |
363 | return PTR_ERR(cr); |
364 | } | |
365 | ||
366 | iotlb_load_cr(obj, cr); | |
367 | kfree(cr); | |
368 | ||
be6d8026 KH |
369 | if (e->prsvd) |
370 | l.base++; | |
a9dcad5e HD |
371 | /* increment victim for next tlb load */ |
372 | if (++l.vict == obj->nr_tlb_entries) | |
be6d8026 | 373 | l.vict = l.base; |
a9dcad5e HD |
374 | iotlb_lock_set(obj, &l); |
375 | out: | |
ebf7cda0 | 376 | pm_runtime_put_sync(obj->dev); |
a9dcad5e HD |
377 | return err; |
378 | } | |
a9dcad5e | 379 | |
5da14a47 OBC |
380 | #else /* !PREFETCH_IOTLB */ |
381 | ||
6c32df43 | 382 | static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
5da14a47 OBC |
383 | { |
384 | return 0; | |
385 | } | |
386 | ||
387 | #endif /* !PREFETCH_IOTLB */ | |
388 | ||
6c32df43 | 389 | static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
5da14a47 OBC |
390 | { |
391 | return load_iotlb_entry(obj, e); | |
392 | } | |
a9dcad5e HD |
393 | |
394 | /** | |
395 | * flush_iotlb_page - Clear an iommu tlb entry | |
396 | * @obj: target iommu | |
397 | * @da: iommu device virtual address | |
398 | * | |
399 | * Clear an iommu tlb entry which includes 'da' address. | |
400 | **/ | |
6c32df43 | 401 | static void flush_iotlb_page(struct omap_iommu *obj, u32 da) |
a9dcad5e | 402 | { |
a9dcad5e | 403 | int i; |
37c2836c | 404 | struct cr_regs cr; |
a9dcad5e | 405 | |
ebf7cda0 | 406 | pm_runtime_get_sync(obj->dev); |
a9dcad5e | 407 | |
37c2836c | 408 | for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) { |
a9dcad5e HD |
409 | u32 start; |
410 | size_t bytes; | |
411 | ||
a9dcad5e HD |
412 | if (!iotlb_cr_valid(&cr)) |
413 | continue; | |
414 | ||
415 | start = iotlb_cr_to_virt(&cr); | |
416 | bytes = iopgsz_to_bytes(cr.cam & 3); | |
417 | ||
418 | if ((start <= da) && (da < start + bytes)) { | |
419 | dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n", | |
420 | __func__, start, da, bytes); | |
0fa035e5 | 421 | iotlb_load_cr(obj, &cr); |
a9dcad5e | 422 | iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); |
f7129a0e | 423 | break; |
a9dcad5e HD |
424 | } |
425 | } | |
ebf7cda0 | 426 | pm_runtime_put_sync(obj->dev); |
a9dcad5e HD |
427 | |
428 | if (i == obj->nr_tlb_entries) | |
429 | dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da); | |
430 | } | |
a9dcad5e HD |
431 | |
432 | /** | |
433 | * flush_iotlb_all - Clear all iommu tlb entries | |
434 | * @obj: target iommu | |
435 | **/ | |
6c32df43 | 436 | static void flush_iotlb_all(struct omap_iommu *obj) |
a9dcad5e HD |
437 | { |
438 | struct iotlb_lock l; | |
439 | ||
ebf7cda0 | 440 | pm_runtime_get_sync(obj->dev); |
a9dcad5e HD |
441 | |
442 | l.base = 0; | |
443 | l.vict = 0; | |
444 | iotlb_lock_set(obj, &l); | |
445 | ||
446 | iommu_write_reg(obj, 1, MMU_GFLUSH); | |
447 | ||
ebf7cda0 | 448 | pm_runtime_put_sync(obj->dev); |
a9dcad5e | 449 | } |
ddfa975a | 450 | |
a9dcad5e HD |
451 | /* |
452 | * H/W pagetable operations | |
453 | */ | |
454 | static void flush_iopgd_range(u32 *first, u32 *last) | |
455 | { | |
456 | /* FIXME: L2 cache should be taken care of if it exists */ | |
457 | do { | |
458 | asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd" | |
459 | : : "r" (first)); | |
460 | first += L1_CACHE_BYTES / sizeof(*first); | |
461 | } while (first <= last); | |
462 | } | |
463 | ||
464 | static void flush_iopte_range(u32 *first, u32 *last) | |
465 | { | |
466 | /* FIXME: L2 cache should be taken care of if it exists */ | |
467 | do { | |
468 | asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte" | |
469 | : : "r" (first)); | |
470 | first += L1_CACHE_BYTES / sizeof(*first); | |
471 | } while (first <= last); | |
472 | } | |
473 | ||
474 | static void iopte_free(u32 *iopte) | |
475 | { | |
476 | /* Note: freed iopte's must be clean ready for re-use */ | |
e28045ab ZZ |
477 | if (iopte) |
478 | kmem_cache_free(iopte_cachep, iopte); | |
a9dcad5e HD |
479 | } |
480 | ||
6c32df43 | 481 | static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da) |
a9dcad5e HD |
482 | { |
483 | u32 *iopte; | |
484 | ||
485 | /* a table has already existed */ | |
486 | if (*iopgd) | |
487 | goto pte_ready; | |
488 | ||
489 | /* | |
490 | * do the allocation outside the page table lock | |
491 | */ | |
492 | spin_unlock(&obj->page_table_lock); | |
493 | iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL); | |
494 | spin_lock(&obj->page_table_lock); | |
495 | ||
496 | if (!*iopgd) { | |
497 | if (!iopte) | |
498 | return ERR_PTR(-ENOMEM); | |
499 | ||
500 | *iopgd = virt_to_phys(iopte) | IOPGD_TABLE; | |
501 | flush_iopgd_range(iopgd, iopgd); | |
502 | ||
503 | dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte); | |
504 | } else { | |
505 | /* We raced, free the reduniovant table */ | |
506 | iopte_free(iopte); | |
507 | } | |
508 | ||
509 | pte_ready: | |
510 | iopte = iopte_offset(iopgd, da); | |
511 | ||
512 | dev_vdbg(obj->dev, | |
513 | "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n", | |
514 | __func__, da, iopgd, *iopgd, iopte, *iopte); | |
515 | ||
516 | return iopte; | |
517 | } | |
518 | ||
6c32df43 | 519 | static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
a9dcad5e HD |
520 | { |
521 | u32 *iopgd = iopgd_offset(obj, da); | |
522 | ||
4abb7617 HD |
523 | if ((da | pa) & ~IOSECTION_MASK) { |
524 | dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", | |
525 | __func__, da, pa, IOSECTION_SIZE); | |
526 | return -EINVAL; | |
527 | } | |
528 | ||
a9dcad5e HD |
529 | *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION; |
530 | flush_iopgd_range(iopgd, iopgd); | |
531 | return 0; | |
532 | } | |
533 | ||
6c32df43 | 534 | static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
a9dcad5e HD |
535 | { |
536 | u32 *iopgd = iopgd_offset(obj, da); | |
537 | int i; | |
538 | ||
4abb7617 HD |
539 | if ((da | pa) & ~IOSUPER_MASK) { |
540 | dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", | |
541 | __func__, da, pa, IOSUPER_SIZE); | |
542 | return -EINVAL; | |
543 | } | |
544 | ||
a9dcad5e HD |
545 | for (i = 0; i < 16; i++) |
546 | *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER; | |
547 | flush_iopgd_range(iopgd, iopgd + 15); | |
548 | return 0; | |
549 | } | |
550 | ||
6c32df43 | 551 | static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
a9dcad5e HD |
552 | { |
553 | u32 *iopgd = iopgd_offset(obj, da); | |
554 | u32 *iopte = iopte_alloc(obj, iopgd, da); | |
555 | ||
556 | if (IS_ERR(iopte)) | |
557 | return PTR_ERR(iopte); | |
558 | ||
559 | *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL; | |
560 | flush_iopte_range(iopte, iopte); | |
561 | ||
562 | dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n", | |
563 | __func__, da, pa, iopte, *iopte); | |
564 | ||
565 | return 0; | |
566 | } | |
567 | ||
6c32df43 | 568 | static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
a9dcad5e HD |
569 | { |
570 | u32 *iopgd = iopgd_offset(obj, da); | |
571 | u32 *iopte = iopte_alloc(obj, iopgd, da); | |
572 | int i; | |
573 | ||
4abb7617 HD |
574 | if ((da | pa) & ~IOLARGE_MASK) { |
575 | dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", | |
576 | __func__, da, pa, IOLARGE_SIZE); | |
577 | return -EINVAL; | |
578 | } | |
579 | ||
a9dcad5e HD |
580 | if (IS_ERR(iopte)) |
581 | return PTR_ERR(iopte); | |
582 | ||
583 | for (i = 0; i < 16; i++) | |
584 | *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE; | |
585 | flush_iopte_range(iopte, iopte + 15); | |
586 | return 0; | |
587 | } | |
588 | ||
6c32df43 OBC |
589 | static int |
590 | iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e) | |
a9dcad5e | 591 | { |
6c32df43 | 592 | int (*fn)(struct omap_iommu *, u32, u32, u32); |
a9dcad5e HD |
593 | u32 prot; |
594 | int err; | |
595 | ||
596 | if (!obj || !e) | |
597 | return -EINVAL; | |
598 | ||
599 | switch (e->pgsz) { | |
600 | case MMU_CAM_PGSZ_16M: | |
601 | fn = iopgd_alloc_super; | |
602 | break; | |
603 | case MMU_CAM_PGSZ_1M: | |
604 | fn = iopgd_alloc_section; | |
605 | break; | |
606 | case MMU_CAM_PGSZ_64K: | |
607 | fn = iopte_alloc_large; | |
608 | break; | |
609 | case MMU_CAM_PGSZ_4K: | |
610 | fn = iopte_alloc_page; | |
611 | break; | |
612 | default: | |
613 | fn = NULL; | |
614 | BUG(); | |
615 | break; | |
616 | } | |
617 | ||
618 | prot = get_iopte_attr(e); | |
619 | ||
620 | spin_lock(&obj->page_table_lock); | |
621 | err = fn(obj, e->da, e->pa, prot); | |
622 | spin_unlock(&obj->page_table_lock); | |
623 | ||
624 | return err; | |
625 | } | |
626 | ||
627 | /** | |
6c32df43 | 628 | * omap_iopgtable_store_entry - Make an iommu pte entry |
a9dcad5e HD |
629 | * @obj: target iommu |
630 | * @e: an iommu tlb entry info | |
631 | **/ | |
4899a563 SA |
632 | static int |
633 | omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e) | |
a9dcad5e HD |
634 | { |
635 | int err; | |
636 | ||
637 | flush_iotlb_page(obj, e->da); | |
638 | err = iopgtable_store_entry_core(obj, e); | |
a9dcad5e | 639 | if (!err) |
5da14a47 | 640 | prefetch_iotlb_entry(obj, e); |
a9dcad5e HD |
641 | return err; |
642 | } | |
a9dcad5e HD |
643 | |
644 | /** | |
645 | * iopgtable_lookup_entry - Lookup an iommu pte entry | |
646 | * @obj: target iommu | |
647 | * @da: iommu device virtual address | |
648 | * @ppgd: iommu pgd entry pointer to be returned | |
649 | * @ppte: iommu pte entry pointer to be returned | |
650 | **/ | |
e1f23813 OBC |
651 | static void |
652 | iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte) | |
a9dcad5e HD |
653 | { |
654 | u32 *iopgd, *iopte = NULL; | |
655 | ||
656 | iopgd = iopgd_offset(obj, da); | |
657 | if (!*iopgd) | |
658 | goto out; | |
659 | ||
a1a54456 | 660 | if (iopgd_is_table(*iopgd)) |
a9dcad5e HD |
661 | iopte = iopte_offset(iopgd, da); |
662 | out: | |
663 | *ppgd = iopgd; | |
664 | *ppte = iopte; | |
665 | } | |
a9dcad5e | 666 | |
6c32df43 | 667 | static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da) |
a9dcad5e HD |
668 | { |
669 | size_t bytes; | |
670 | u32 *iopgd = iopgd_offset(obj, da); | |
671 | int nent = 1; | |
672 | ||
673 | if (!*iopgd) | |
674 | return 0; | |
675 | ||
a1a54456 | 676 | if (iopgd_is_table(*iopgd)) { |
a9dcad5e HD |
677 | int i; |
678 | u32 *iopte = iopte_offset(iopgd, da); | |
679 | ||
680 | bytes = IOPTE_SIZE; | |
681 | if (*iopte & IOPTE_LARGE) { | |
682 | nent *= 16; | |
683 | /* rewind to the 1st entry */ | |
c127c7dc | 684 | iopte = iopte_offset(iopgd, (da & IOLARGE_MASK)); |
a9dcad5e HD |
685 | } |
686 | bytes *= nent; | |
687 | memset(iopte, 0, nent * sizeof(*iopte)); | |
688 | flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte)); | |
689 | ||
690 | /* | |
691 | * do table walk to check if this table is necessary or not | |
692 | */ | |
693 | iopte = iopte_offset(iopgd, 0); | |
694 | for (i = 0; i < PTRS_PER_IOPTE; i++) | |
695 | if (iopte[i]) | |
696 | goto out; | |
697 | ||
698 | iopte_free(iopte); | |
699 | nent = 1; /* for the next L1 entry */ | |
700 | } else { | |
701 | bytes = IOPGD_SIZE; | |
dcc730dc | 702 | if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) { |
a9dcad5e HD |
703 | nent *= 16; |
704 | /* rewind to the 1st entry */ | |
8d33ea58 | 705 | iopgd = iopgd_offset(obj, (da & IOSUPER_MASK)); |
a9dcad5e HD |
706 | } |
707 | bytes *= nent; | |
708 | } | |
709 | memset(iopgd, 0, nent * sizeof(*iopgd)); | |
710 | flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd)); | |
711 | out: | |
712 | return bytes; | |
713 | } | |
714 | ||
715 | /** | |
716 | * iopgtable_clear_entry - Remove an iommu pte entry | |
717 | * @obj: target iommu | |
718 | * @da: iommu device virtual address | |
719 | **/ | |
6c32df43 | 720 | static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da) |
a9dcad5e HD |
721 | { |
722 | size_t bytes; | |
723 | ||
724 | spin_lock(&obj->page_table_lock); | |
725 | ||
726 | bytes = iopgtable_clear_entry_core(obj, da); | |
727 | flush_iotlb_page(obj, da); | |
728 | ||
729 | spin_unlock(&obj->page_table_lock); | |
730 | ||
731 | return bytes; | |
732 | } | |
a9dcad5e | 733 | |
6c32df43 | 734 | static void iopgtable_clear_entry_all(struct omap_iommu *obj) |
a9dcad5e HD |
735 | { |
736 | int i; | |
737 | ||
738 | spin_lock(&obj->page_table_lock); | |
739 | ||
740 | for (i = 0; i < PTRS_PER_IOPGD; i++) { | |
741 | u32 da; | |
742 | u32 *iopgd; | |
743 | ||
744 | da = i << IOPGD_SHIFT; | |
745 | iopgd = iopgd_offset(obj, da); | |
746 | ||
747 | if (!*iopgd) | |
748 | continue; | |
749 | ||
a1a54456 | 750 | if (iopgd_is_table(*iopgd)) |
a9dcad5e HD |
751 | iopte_free(iopte_offset(iopgd, 0)); |
752 | ||
753 | *iopgd = 0; | |
754 | flush_iopgd_range(iopgd, iopgd); | |
755 | } | |
756 | ||
757 | flush_iotlb_all(obj); | |
758 | ||
759 | spin_unlock(&obj->page_table_lock); | |
760 | } | |
761 | ||
762 | /* | |
763 | * Device IOMMU generic operations | |
764 | */ | |
765 | static irqreturn_t iommu_fault_handler(int irq, void *data) | |
766 | { | |
d594f1f3 | 767 | u32 da, errs; |
a9dcad5e | 768 | u32 *iopgd, *iopte; |
6c32df43 | 769 | struct omap_iommu *obj = data; |
e7f10f02 | 770 | struct iommu_domain *domain = obj->domain; |
8cf851e0 | 771 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
a9dcad5e | 772 | |
2088ecba | 773 | if (!omap_domain->iommu_dev) |
a9dcad5e HD |
774 | return IRQ_NONE; |
775 | ||
d594f1f3 | 776 | errs = iommu_report_fault(obj, &da); |
c56b2ddd LP |
777 | if (errs == 0) |
778 | return IRQ_HANDLED; | |
d594f1f3 DC |
779 | |
780 | /* Fault callback or TLB/PTE Dynamic loading */ | |
e7f10f02 | 781 | if (!report_iommu_fault(domain, obj->dev, da, 0)) |
a9dcad5e HD |
782 | return IRQ_HANDLED; |
783 | ||
37b29810 HD |
784 | iommu_disable(obj); |
785 | ||
a9dcad5e HD |
786 | iopgd = iopgd_offset(obj, da); |
787 | ||
a1a54456 | 788 | if (!iopgd_is_table(*iopgd)) { |
b6c2e09f SA |
789 | dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n", |
790 | obj->name, errs, da, iopgd, *iopgd); | |
a9dcad5e HD |
791 | return IRQ_NONE; |
792 | } | |
793 | ||
794 | iopte = iopte_offset(iopgd, da); | |
795 | ||
b6c2e09f SA |
796 | dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n", |
797 | obj->name, errs, da, iopgd, *iopgd, iopte, *iopte); | |
a9dcad5e HD |
798 | |
799 | return IRQ_NONE; | |
800 | } | |
801 | ||
802 | static int device_match_by_alias(struct device *dev, void *data) | |
803 | { | |
6c32df43 | 804 | struct omap_iommu *obj = to_iommu(dev); |
a9dcad5e HD |
805 | const char *name = data; |
806 | ||
807 | pr_debug("%s: %s %s\n", __func__, obj->name, name); | |
808 | ||
809 | return strcmp(obj->name, name) == 0; | |
810 | } | |
811 | ||
812 | /** | |
f626b52d | 813 | * omap_iommu_attach() - attach iommu device to an iommu domain |
fabdbca8 | 814 | * @name: name of target omap iommu device |
f626b52d | 815 | * @iopgd: page table |
a9dcad5e | 816 | **/ |
fabdbca8 | 817 | static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd) |
a9dcad5e | 818 | { |
7ee08b9e | 819 | int err; |
fabdbca8 OBC |
820 | struct device *dev; |
821 | struct omap_iommu *obj; | |
822 | ||
823 | dev = driver_find_device(&omap_iommu_driver.driver, NULL, | |
824 | (void *)name, | |
825 | device_match_by_alias); | |
826 | if (!dev) | |
7ee08b9e | 827 | return ERR_PTR(-ENODEV); |
fabdbca8 OBC |
828 | |
829 | obj = to_iommu(dev); | |
a9dcad5e | 830 | |
f626b52d | 831 | spin_lock(&obj->iommu_lock); |
a9dcad5e | 832 | |
f626b52d OBC |
833 | obj->iopgd = iopgd; |
834 | err = iommu_enable(obj); | |
835 | if (err) | |
836 | goto err_enable; | |
837 | flush_iotlb_all(obj); | |
838 | ||
f626b52d | 839 | spin_unlock(&obj->iommu_lock); |
a9dcad5e HD |
840 | |
841 | dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); | |
842 | return obj; | |
843 | ||
a9dcad5e | 844 | err_enable: |
f626b52d | 845 | spin_unlock(&obj->iommu_lock); |
a9dcad5e HD |
846 | return ERR_PTR(err); |
847 | } | |
a9dcad5e HD |
848 | |
849 | /** | |
f626b52d | 850 | * omap_iommu_detach - release iommu device |
a9dcad5e HD |
851 | * @obj: target iommu |
852 | **/ | |
6c32df43 | 853 | static void omap_iommu_detach(struct omap_iommu *obj) |
a9dcad5e | 854 | { |
acf9d467 | 855 | if (!obj || IS_ERR(obj)) |
a9dcad5e HD |
856 | return; |
857 | ||
f626b52d | 858 | spin_lock(&obj->iommu_lock); |
a9dcad5e | 859 | |
2088ecba | 860 | iommu_disable(obj); |
f626b52d | 861 | obj->iopgd = NULL; |
d594f1f3 | 862 | |
f626b52d | 863 | spin_unlock(&obj->iommu_lock); |
d594f1f3 | 864 | |
a9dcad5e | 865 | dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); |
d594f1f3 | 866 | } |
d594f1f3 | 867 | |
a9dcad5e HD |
868 | /* |
869 | * OMAP Device MMU(IOMMU) detection | |
870 | */ | |
d34d6517 | 871 | static int omap_iommu_probe(struct platform_device *pdev) |
a9dcad5e HD |
872 | { |
873 | int err = -ENODEV; | |
a9dcad5e | 874 | int irq; |
6c32df43 | 875 | struct omap_iommu *obj; |
a9dcad5e | 876 | struct resource *res; |
99cb9aee | 877 | struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev); |
3c92748d | 878 | struct device_node *of = pdev->dev.of_node; |
a9dcad5e | 879 | |
f129b3df | 880 | obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL); |
a9dcad5e HD |
881 | if (!obj) |
882 | return -ENOMEM; | |
883 | ||
3c92748d FV |
884 | if (of) { |
885 | obj->name = dev_name(&pdev->dev); | |
886 | obj->nr_tlb_entries = 32; | |
887 | err = of_property_read_u32(of, "ti,#tlb-entries", | |
888 | &obj->nr_tlb_entries); | |
889 | if (err && err != -EINVAL) | |
890 | return err; | |
891 | if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8) | |
892 | return -EINVAL; | |
b148d5fb SA |
893 | if (of_find_property(of, "ti,iommu-bus-err-back", NULL)) |
894 | obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN; | |
3c92748d FV |
895 | } else { |
896 | obj->nr_tlb_entries = pdata->nr_tlb_entries; | |
897 | obj->name = pdata->name; | |
3c92748d | 898 | } |
3c92748d | 899 | |
a9dcad5e HD |
900 | obj->dev = &pdev->dev; |
901 | obj->ctx = (void *)obj + sizeof(*obj); | |
902 | ||
f626b52d | 903 | spin_lock_init(&obj->iommu_lock); |
a9dcad5e | 904 | spin_lock_init(&obj->page_table_lock); |
a9dcad5e HD |
905 | |
906 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
f129b3df SA |
907 | obj->regbase = devm_ioremap_resource(obj->dev, res); |
908 | if (IS_ERR(obj->regbase)) | |
909 | return PTR_ERR(obj->regbase); | |
da4a0f76 | 910 | |
a9dcad5e | 911 | irq = platform_get_irq(pdev, 0); |
f129b3df SA |
912 | if (irq < 0) |
913 | return -ENODEV; | |
914 | ||
915 | err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED, | |
916 | dev_name(obj->dev), obj); | |
a9dcad5e | 917 | if (err < 0) |
f129b3df | 918 | return err; |
a9dcad5e HD |
919 | platform_set_drvdata(pdev, obj); |
920 | ||
ebf7cda0 ORL |
921 | pm_runtime_irq_safe(obj->dev); |
922 | pm_runtime_enable(obj->dev); | |
923 | ||
61c75352 SA |
924 | omap_iommu_debugfs_add(obj); |
925 | ||
a9dcad5e HD |
926 | dev_info(&pdev->dev, "%s registered\n", obj->name); |
927 | return 0; | |
a9dcad5e HD |
928 | } |
929 | ||
d34d6517 | 930 | static int omap_iommu_remove(struct platform_device *pdev) |
a9dcad5e | 931 | { |
6c32df43 | 932 | struct omap_iommu *obj = platform_get_drvdata(pdev); |
a9dcad5e | 933 | |
a9dcad5e | 934 | iopgtable_clear_entry_all(obj); |
61c75352 | 935 | omap_iommu_debugfs_remove(obj); |
a9dcad5e | 936 | |
ebf7cda0 ORL |
937 | pm_runtime_disable(obj->dev); |
938 | ||
a9dcad5e | 939 | dev_info(&pdev->dev, "%s removed\n", obj->name); |
a9dcad5e HD |
940 | return 0; |
941 | } | |
942 | ||
d943b0ff | 943 | static const struct of_device_id omap_iommu_of_match[] = { |
3c92748d FV |
944 | { .compatible = "ti,omap2-iommu" }, |
945 | { .compatible = "ti,omap4-iommu" }, | |
946 | { .compatible = "ti,dra7-iommu" }, | |
947 | {}, | |
948 | }; | |
3c92748d | 949 | |
a9dcad5e HD |
950 | static struct platform_driver omap_iommu_driver = { |
951 | .probe = omap_iommu_probe, | |
d34d6517 | 952 | .remove = omap_iommu_remove, |
a9dcad5e HD |
953 | .driver = { |
954 | .name = "omap-iommu", | |
3c92748d | 955 | .of_match_table = of_match_ptr(omap_iommu_of_match), |
a9dcad5e HD |
956 | }, |
957 | }; | |
958 | ||
959 | static void iopte_cachep_ctor(void *iopte) | |
960 | { | |
961 | clean_dcache_area(iopte, IOPTE_TABLE_SIZE); | |
962 | } | |
963 | ||
286f600b | 964 | static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz) |
ed1c7de2 TL |
965 | { |
966 | memset(e, 0, sizeof(*e)); | |
967 | ||
968 | e->da = da; | |
969 | e->pa = pa; | |
d760e3e0 | 970 | e->valid = MMU_CAM_V; |
286f600b LP |
971 | e->pgsz = pgsz; |
972 | e->endian = MMU_RAM_ENDIAN_LITTLE; | |
973 | e->elsz = MMU_RAM_ELSZ_8; | |
974 | e->mixed = 0; | |
ed1c7de2 TL |
975 | |
976 | return iopgsz_to_bytes(e->pgsz); | |
977 | } | |
978 | ||
f626b52d | 979 | static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, |
5009065d | 980 | phys_addr_t pa, size_t bytes, int prot) |
f626b52d | 981 | { |
8cf851e0 | 982 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
6c32df43 | 983 | struct omap_iommu *oiommu = omap_domain->iommu_dev; |
f626b52d | 984 | struct device *dev = oiommu->dev; |
f626b52d OBC |
985 | struct iotlb_entry e; |
986 | int omap_pgsz; | |
286f600b | 987 | u32 ret; |
f626b52d | 988 | |
f626b52d OBC |
989 | omap_pgsz = bytes_to_iopgsz(bytes); |
990 | if (omap_pgsz < 0) { | |
991 | dev_err(dev, "invalid size to map: %d\n", bytes); | |
992 | return -EINVAL; | |
993 | } | |
994 | ||
1d7f449c | 995 | dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%x\n", da, &pa, bytes); |
f626b52d | 996 | |
286f600b | 997 | iotlb_init_entry(&e, da, pa, omap_pgsz); |
f626b52d | 998 | |
6c32df43 | 999 | ret = omap_iopgtable_store_entry(oiommu, &e); |
b4550d41 | 1000 | if (ret) |
6c32df43 | 1001 | dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret); |
f626b52d | 1002 | |
b4550d41 | 1003 | return ret; |
f626b52d OBC |
1004 | } |
1005 | ||
5009065d OBC |
1006 | static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, |
1007 | size_t size) | |
f626b52d | 1008 | { |
8cf851e0 | 1009 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
6c32df43 | 1010 | struct omap_iommu *oiommu = omap_domain->iommu_dev; |
f626b52d | 1011 | struct device *dev = oiommu->dev; |
f626b52d | 1012 | |
5009065d | 1013 | dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size); |
f626b52d | 1014 | |
5009065d | 1015 | return iopgtable_clear_entry(oiommu, da); |
f626b52d OBC |
1016 | } |
1017 | ||
1018 | static int | |
1019 | omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) | |
1020 | { | |
8cf851e0 | 1021 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
6c32df43 | 1022 | struct omap_iommu *oiommu; |
fabdbca8 | 1023 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
f626b52d OBC |
1024 | int ret = 0; |
1025 | ||
e3f595b9 SA |
1026 | if (!arch_data || !arch_data->name) { |
1027 | dev_err(dev, "device doesn't have an associated iommu\n"); | |
1028 | return -EINVAL; | |
1029 | } | |
1030 | ||
f626b52d OBC |
1031 | spin_lock(&omap_domain->lock); |
1032 | ||
1033 | /* only a single device is supported per domain for now */ | |
1034 | if (omap_domain->iommu_dev) { | |
1035 | dev_err(dev, "iommu domain is already attached\n"); | |
1036 | ret = -EBUSY; | |
1037 | goto out; | |
1038 | } | |
1039 | ||
1040 | /* get a handle to and enable the omap iommu */ | |
fabdbca8 | 1041 | oiommu = omap_iommu_attach(arch_data->name, omap_domain->pgtable); |
f626b52d OBC |
1042 | if (IS_ERR(oiommu)) { |
1043 | ret = PTR_ERR(oiommu); | |
1044 | dev_err(dev, "can't get omap iommu: %d\n", ret); | |
1045 | goto out; | |
1046 | } | |
1047 | ||
fabdbca8 | 1048 | omap_domain->iommu_dev = arch_data->iommu_dev = oiommu; |
803b5277 | 1049 | omap_domain->dev = dev; |
e7f10f02 | 1050 | oiommu->domain = domain; |
f626b52d OBC |
1051 | |
1052 | out: | |
1053 | spin_unlock(&omap_domain->lock); | |
1054 | return ret; | |
1055 | } | |
1056 | ||
803b5277 ORL |
1057 | static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, |
1058 | struct device *dev) | |
f626b52d | 1059 | { |
fabdbca8 | 1060 | struct omap_iommu *oiommu = dev_to_omap_iommu(dev); |
803b5277 | 1061 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
f626b52d OBC |
1062 | |
1063 | /* only a single device is supported per domain for now */ | |
1064 | if (omap_domain->iommu_dev != oiommu) { | |
1065 | dev_err(dev, "invalid iommu device\n"); | |
803b5277 | 1066 | return; |
f626b52d OBC |
1067 | } |
1068 | ||
1069 | iopgtable_clear_entry_all(oiommu); | |
1070 | ||
1071 | omap_iommu_detach(oiommu); | |
1072 | ||
fabdbca8 | 1073 | omap_domain->iommu_dev = arch_data->iommu_dev = NULL; |
803b5277 | 1074 | omap_domain->dev = NULL; |
f24d9ad3 | 1075 | oiommu->domain = NULL; |
803b5277 | 1076 | } |
f626b52d | 1077 | |
803b5277 ORL |
1078 | static void omap_iommu_detach_dev(struct iommu_domain *domain, |
1079 | struct device *dev) | |
1080 | { | |
8cf851e0 | 1081 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
803b5277 ORL |
1082 | |
1083 | spin_lock(&omap_domain->lock); | |
1084 | _omap_iommu_detach_dev(omap_domain, dev); | |
f626b52d OBC |
1085 | spin_unlock(&omap_domain->lock); |
1086 | } | |
1087 | ||
8cf851e0 | 1088 | static struct iommu_domain *omap_iommu_domain_alloc(unsigned type) |
f626b52d OBC |
1089 | { |
1090 | struct omap_iommu_domain *omap_domain; | |
1091 | ||
8cf851e0 JR |
1092 | if (type != IOMMU_DOMAIN_UNMANAGED) |
1093 | return NULL; | |
1094 | ||
f626b52d OBC |
1095 | omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL); |
1096 | if (!omap_domain) { | |
1097 | pr_err("kzalloc failed\n"); | |
1098 | goto out; | |
1099 | } | |
1100 | ||
1101 | omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL); | |
1102 | if (!omap_domain->pgtable) { | |
1103 | pr_err("kzalloc failed\n"); | |
1104 | goto fail_nomem; | |
1105 | } | |
1106 | ||
1107 | /* | |
1108 | * should never fail, but please keep this around to ensure | |
1109 | * we keep the hardware happy | |
1110 | */ | |
1111 | BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE)); | |
1112 | ||
1113 | clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE); | |
1114 | spin_lock_init(&omap_domain->lock); | |
1115 | ||
8cf851e0 JR |
1116 | omap_domain->domain.geometry.aperture_start = 0; |
1117 | omap_domain->domain.geometry.aperture_end = (1ULL << 32) - 1; | |
1118 | omap_domain->domain.geometry.force_aperture = true; | |
f626b52d | 1119 | |
8cf851e0 | 1120 | return &omap_domain->domain; |
f626b52d OBC |
1121 | |
1122 | fail_nomem: | |
1123 | kfree(omap_domain); | |
1124 | out: | |
8cf851e0 | 1125 | return NULL; |
f626b52d OBC |
1126 | } |
1127 | ||
8cf851e0 | 1128 | static void omap_iommu_domain_free(struct iommu_domain *domain) |
f626b52d | 1129 | { |
8cf851e0 | 1130 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
f626b52d | 1131 | |
803b5277 ORL |
1132 | /* |
1133 | * An iommu device is still attached | |
1134 | * (currently, only one device can be attached) ? | |
1135 | */ | |
1136 | if (omap_domain->iommu_dev) | |
1137 | _omap_iommu_detach_dev(omap_domain, omap_domain->dev); | |
1138 | ||
f626b52d OBC |
1139 | kfree(omap_domain->pgtable); |
1140 | kfree(omap_domain); | |
1141 | } | |
1142 | ||
1143 | static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, | |
bb5547ac | 1144 | dma_addr_t da) |
f626b52d | 1145 | { |
8cf851e0 | 1146 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
6c32df43 | 1147 | struct omap_iommu *oiommu = omap_domain->iommu_dev; |
f626b52d OBC |
1148 | struct device *dev = oiommu->dev; |
1149 | u32 *pgd, *pte; | |
1150 | phys_addr_t ret = 0; | |
1151 | ||
1152 | iopgtable_lookup_entry(oiommu, da, &pgd, &pte); | |
1153 | ||
1154 | if (pte) { | |
1155 | if (iopte_is_small(*pte)) | |
1156 | ret = omap_iommu_translate(*pte, da, IOPTE_MASK); | |
1157 | else if (iopte_is_large(*pte)) | |
1158 | ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); | |
1159 | else | |
2abfcfbc SA |
1160 | dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte, |
1161 | (unsigned long long)da); | |
f626b52d OBC |
1162 | } else { |
1163 | if (iopgd_is_section(*pgd)) | |
1164 | ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); | |
1165 | else if (iopgd_is_super(*pgd)) | |
1166 | ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); | |
1167 | else | |
2abfcfbc SA |
1168 | dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd, |
1169 | (unsigned long long)da); | |
f626b52d OBC |
1170 | } |
1171 | ||
1172 | return ret; | |
1173 | } | |
1174 | ||
07a02030 LP |
1175 | static int omap_iommu_add_device(struct device *dev) |
1176 | { | |
1177 | struct omap_iommu_arch_data *arch_data; | |
1178 | struct device_node *np; | |
7d682774 | 1179 | struct platform_device *pdev; |
07a02030 LP |
1180 | |
1181 | /* | |
1182 | * Allocate the archdata iommu structure for DT-based devices. | |
1183 | * | |
1184 | * TODO: Simplify this when removing non-DT support completely from the | |
1185 | * IOMMU users. | |
1186 | */ | |
1187 | if (!dev->of_node) | |
1188 | return 0; | |
1189 | ||
1190 | np = of_parse_phandle(dev->of_node, "iommus", 0); | |
1191 | if (!np) | |
1192 | return 0; | |
1193 | ||
7d682774 SA |
1194 | pdev = of_find_device_by_node(np); |
1195 | if (WARN_ON(!pdev)) { | |
1196 | of_node_put(np); | |
1197 | return -EINVAL; | |
1198 | } | |
1199 | ||
07a02030 LP |
1200 | arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL); |
1201 | if (!arch_data) { | |
1202 | of_node_put(np); | |
1203 | return -ENOMEM; | |
1204 | } | |
1205 | ||
7d682774 | 1206 | arch_data->name = kstrdup(dev_name(&pdev->dev), GFP_KERNEL); |
07a02030 LP |
1207 | dev->archdata.iommu = arch_data; |
1208 | ||
1209 | of_node_put(np); | |
1210 | ||
1211 | return 0; | |
1212 | } | |
1213 | ||
1214 | static void omap_iommu_remove_device(struct device *dev) | |
1215 | { | |
1216 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; | |
1217 | ||
1218 | if (!dev->of_node || !arch_data) | |
1219 | return; | |
1220 | ||
1221 | kfree(arch_data->name); | |
1222 | kfree(arch_data); | |
1223 | } | |
1224 | ||
b22f6434 | 1225 | static const struct iommu_ops omap_iommu_ops = { |
8cf851e0 JR |
1226 | .domain_alloc = omap_iommu_domain_alloc, |
1227 | .domain_free = omap_iommu_domain_free, | |
f626b52d OBC |
1228 | .attach_dev = omap_iommu_attach_dev, |
1229 | .detach_dev = omap_iommu_detach_dev, | |
1230 | .map = omap_iommu_map, | |
1231 | .unmap = omap_iommu_unmap, | |
315786eb | 1232 | .map_sg = default_iommu_map_sg, |
f626b52d | 1233 | .iova_to_phys = omap_iommu_iova_to_phys, |
07a02030 LP |
1234 | .add_device = omap_iommu_add_device, |
1235 | .remove_device = omap_iommu_remove_device, | |
66bc8cf3 | 1236 | .pgsize_bitmap = OMAP_IOMMU_PGSIZES, |
f626b52d OBC |
1237 | }; |
1238 | ||
a9dcad5e HD |
1239 | static int __init omap_iommu_init(void) |
1240 | { | |
1241 | struct kmem_cache *p; | |
1242 | const unsigned long flags = SLAB_HWCACHE_ALIGN; | |
1243 | size_t align = 1 << 10; /* L2 pagetable alignement */ | |
f938aab2 TR |
1244 | struct device_node *np; |
1245 | ||
1246 | np = of_find_matching_node(NULL, omap_iommu_of_match); | |
1247 | if (!np) | |
1248 | return 0; | |
1249 | ||
1250 | of_node_put(np); | |
a9dcad5e HD |
1251 | |
1252 | p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags, | |
1253 | iopte_cachep_ctor); | |
1254 | if (!p) | |
1255 | return -ENOMEM; | |
1256 | iopte_cachep = p; | |
1257 | ||
a65bc64f | 1258 | bus_set_iommu(&platform_bus_type, &omap_iommu_ops); |
f626b52d | 1259 | |
61c75352 SA |
1260 | omap_iommu_debugfs_init(); |
1261 | ||
a9dcad5e HD |
1262 | return platform_driver_register(&omap_iommu_driver); |
1263 | } | |
435792d9 | 1264 | subsys_initcall(omap_iommu_init); |
0cdbf727 | 1265 | /* must be ready before omap3isp is probed */ |