Commit | Line | Data |
---|---|---|
9b6b563c PM |
1 | /* |
2 | * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation | |
3 | * Rewrite, cleanup: | |
91f14480 | 4 | * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation |
2db4928b | 5 | * |
9b6b563c PM |
6 | * This program is free software; you can redistribute it and/or modify |
7 | * it under the terms of the GNU General Public License as published by | |
8 | * the Free Software Foundation; either version 2 of the License, or | |
9 | * (at your option) any later version. | |
2db4928b | 10 | * |
9b6b563c PM |
11 | * This program is distributed in the hope that it will be useful, |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
2db4928b | 15 | * |
9b6b563c PM |
16 | * You should have received a copy of the GNU General Public License |
17 | * along with this program; if not, write to the Free Software | |
18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
19 | */ | |
20 | ||
21 | #ifndef _ASM_IOMMU_H | |
22 | #define _ASM_IOMMU_H | |
88ced031 | 23 | #ifdef __KERNEL__ |
9b6b563c | 24 | |
5d2efba6 | 25 | #include <linux/compiler.h> |
9b6b563c PM |
26 | #include <linux/spinlock.h> |
27 | #include <linux/device.h> | |
28 | #include <linux/dma-mapping.h> | |
1977f032 | 29 | #include <linux/bitops.h> |
7e11580b | 30 | #include <asm/machdep.h> |
5d2efba6 | 31 | #include <asm/types.h> |
798248a3 | 32 | #include <asm/pci-bridge.h> |
5d2efba6 | 33 | |
e589a440 AP |
34 | #define IOMMU_PAGE_SHIFT_4K 12 |
35 | #define IOMMU_PAGE_SIZE_4K (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K) | |
36 | #define IOMMU_PAGE_MASK_4K (~((1 << IOMMU_PAGE_SHIFT_4K) - 1)) | |
37 | #define IOMMU_PAGE_ALIGN_4K(addr) _ALIGN_UP(addr, IOMMU_PAGE_SIZE_4K) | |
5d2efba6 | 38 | |
d0847757 AP |
39 | #define IOMMU_PAGE_SIZE(tblptr) (ASM_CONST(1) << (tblptr)->it_page_shift) |
40 | #define IOMMU_PAGE_MASK(tblptr) (~((1 << (tblptr)->it_page_shift) - 1)) | |
41 | #define IOMMU_PAGE_ALIGN(addr, tblptr) _ALIGN_UP(addr, IOMMU_PAGE_SIZE(tblptr)) | |
42 | ||
165785e5 JK |
43 | /* Boot time flags */ |
44 | extern int iommu_is_off; | |
45 | extern int iommu_force_on; | |
5d2efba6 | 46 | |
da004c36 | 47 | struct iommu_table_ops { |
05c6cfb9 AK |
48 | /* |
49 | * When called with direction==DMA_NONE, it is equal to clear(). | |
50 | * uaddr is a linear map address. | |
51 | */ | |
da004c36 AK |
52 | int (*set)(struct iommu_table *tbl, |
53 | long index, long npages, | |
54 | unsigned long uaddr, | |
55 | enum dma_data_direction direction, | |
56 | struct dma_attrs *attrs); | |
05c6cfb9 AK |
57 | #ifdef CONFIG_IOMMU_API |
58 | /* | |
59 | * Exchanges existing TCE with new TCE plus direction bits; | |
60 | * returns old TCE and DMA direction mask. | |
61 | * @tce is a physical address. | |
62 | */ | |
63 | int (*exchange)(struct iommu_table *tbl, | |
64 | long index, | |
65 | unsigned long *hpa, | |
66 | enum dma_data_direction *direction); | |
67 | #endif | |
da004c36 AK |
68 | void (*clear)(struct iommu_table *tbl, |
69 | long index, long npages); | |
05c6cfb9 | 70 | /* get() returns a physical address */ |
da004c36 AK |
71 | unsigned long (*get)(struct iommu_table *tbl, long index); |
72 | void (*flush)(struct iommu_table *tbl); | |
4793d65d | 73 | void (*free)(struct iommu_table *tbl); |
da004c36 AK |
74 | }; |
75 | ||
76 | /* These are used by VIO */ | |
77 | extern struct iommu_table_ops iommu_table_lpar_multi_ops; | |
78 | extern struct iommu_table_ops iommu_table_pseries_ops; | |
79 | ||
9b6b563c PM |
80 | /* |
81 | * IOMAP_MAX_ORDER defines the largest contiguous block | |
82 | * of dma space we can get. IOMAP_MAX_ORDER = 13 | |
83 | * allows up to 2**12 pages (4096 * 4096) = 16 MB | |
84 | */ | |
5d2efba6 | 85 | #define IOMAP_MAX_ORDER 13 |
9b6b563c | 86 | |
b4c3a872 AB |
87 | #define IOMMU_POOL_HASHBITS 2 |
88 | #define IOMMU_NR_POOLS (1 << IOMMU_POOL_HASHBITS) | |
89 | ||
90 | struct iommu_pool { | |
91 | unsigned long start; | |
92 | unsigned long end; | |
93 | unsigned long hint; | |
94 | spinlock_t lock; | |
95 | } ____cacheline_aligned_in_smp; | |
96 | ||
9b6b563c PM |
97 | struct iommu_table { |
98 | unsigned long it_busno; /* Bus number this table belongs to */ | |
99 | unsigned long it_size; /* Size of iommu table in entries */ | |
bbb845c4 AK |
100 | unsigned long it_indirect_levels; |
101 | unsigned long it_level_size; | |
00547193 | 102 | unsigned long it_allocated_size; |
9b6b563c PM |
103 | unsigned long it_offset; /* Offset into global table */ |
104 | unsigned long it_base; /* mapped address of tce table */ | |
105 | unsigned long it_index; /* which iommu table this is */ | |
106 | unsigned long it_type; /* type: PCI or Virtual Bus */ | |
107 | unsigned long it_blocksize; /* Entries in each block (cacheline) */ | |
b4c3a872 AB |
108 | unsigned long poolsize; |
109 | unsigned long nr_pools; | |
110 | struct iommu_pool large_pool; | |
111 | struct iommu_pool pools[IOMMU_NR_POOLS]; | |
9b6b563c | 112 | unsigned long *it_map; /* A simple allocation bitmap for now */ |
3a553170 | 113 | unsigned long it_page_shift;/* table iommu page size */ |
0eaf4def | 114 | struct list_head it_group_list;/* List of iommu_table_group_link */ |
2157e7b8 | 115 | unsigned long *it_userspace; /* userspace view of the table */ |
da004c36 | 116 | struct iommu_table_ops *it_ops; |
9b6b563c PM |
117 | }; |
118 | ||
2157e7b8 AK |
119 | #define IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry) \ |
120 | ((tbl)->it_userspace ? \ | |
121 | &((tbl)->it_userspace[(entry) - (tbl)->it_offset]) : \ | |
122 | NULL) | |
123 | ||
d0847757 AP |
124 | /* Pure 2^n version of get_order */ |
125 | static inline __attribute_const__ | |
126 | int get_iommu_order(unsigned long size, struct iommu_table *tbl) | |
127 | { | |
128 | return __ilog2((size - 1) >> tbl->it_page_shift) + 1; | |
129 | } | |
130 | ||
131 | ||
9b6b563c | 132 | struct scatterlist; |
9b6b563c | 133 | |
2db4928b BH |
134 | #ifdef CONFIG_PPC64 |
135 | ||
136 | static inline void set_iommu_table_base(struct device *dev, | |
137 | struct iommu_table *base) | |
738ef42e | 138 | { |
2db4928b | 139 | dev->archdata.iommu_table_base = base; |
738ef42e BB |
140 | } |
141 | ||
142 | static inline void *get_iommu_table_base(struct device *dev) | |
143 | { | |
2db4928b | 144 | return dev->archdata.iommu_table_base; |
738ef42e BB |
145 | } |
146 | ||
2db4928b BH |
147 | extern int dma_iommu_dma_supported(struct device *dev, u64 mask); |
148 | ||
9b6b563c | 149 | /* Frees table for an individual device node */ |
68d315f5 | 150 | extern void iommu_free_table(struct iommu_table *tbl, const char *node_name); |
9b6b563c | 151 | |
9b6b563c PM |
152 | /* Initializes an iommu_table based in values set in the passed-in |
153 | * structure | |
154 | */ | |
ca1588e7 AB |
155 | extern struct iommu_table *iommu_init_table(struct iommu_table * tbl, |
156 | int nid); | |
e633bc86 | 157 | #define IOMMU_TABLE_GROUP_MAX_TABLES 2 |
b348aa65 | 158 | |
f87a8864 AK |
159 | struct iommu_table_group; |
160 | ||
161 | struct iommu_table_group_ops { | |
00547193 AK |
162 | unsigned long (*get_table_size)( |
163 | __u32 page_shift, | |
164 | __u64 window_size, | |
165 | __u32 levels); | |
4793d65d AK |
166 | long (*create_table)(struct iommu_table_group *table_group, |
167 | int num, | |
168 | __u32 page_shift, | |
169 | __u64 window_size, | |
170 | __u32 levels, | |
171 | struct iommu_table **ptbl); | |
172 | long (*set_window)(struct iommu_table_group *table_group, | |
173 | int num, | |
174 | struct iommu_table *tblnew); | |
175 | long (*unset_window)(struct iommu_table_group *table_group, | |
176 | int num); | |
f87a8864 AK |
177 | /* Switch ownership from platform code to external user (e.g. VFIO) */ |
178 | void (*take_ownership)(struct iommu_table_group *table_group); | |
179 | /* Switch ownership from external user (e.g. VFIO) back to core */ | |
180 | void (*release_ownership)(struct iommu_table_group *table_group); | |
181 | }; | |
182 | ||
0eaf4def AK |
183 | struct iommu_table_group_link { |
184 | struct list_head next; | |
185 | struct rcu_head rcu; | |
186 | struct iommu_table_group *table_group; | |
187 | }; | |
188 | ||
b348aa65 | 189 | struct iommu_table_group { |
4793d65d AK |
190 | /* IOMMU properties */ |
191 | __u32 tce32_start; | |
192 | __u32 tce32_size; | |
193 | __u64 pgsizes; /* Bitmap of supported page sizes */ | |
194 | __u32 max_dynamic_windows_supported; | |
195 | __u32 max_levels; | |
196 | ||
b348aa65 AK |
197 | struct iommu_group *group; |
198 | struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES]; | |
f87a8864 | 199 | struct iommu_table_group_ops *ops; |
b348aa65 AK |
200 | }; |
201 | ||
d905c5df | 202 | #ifdef CONFIG_IOMMU_API |
b348aa65 AK |
203 | |
204 | extern void iommu_register_group(struct iommu_table_group *table_group, | |
4e13c1ac | 205 | int pci_domain_number, unsigned long pe_num); |
d905c5df AK |
206 | extern int iommu_add_device(struct device *dev); |
207 | extern void iommu_del_device(struct device *dev); | |
4ad04e59 | 208 | extern int __init tce_iommu_bus_notifier_init(void); |
05c6cfb9 AK |
209 | extern long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry, |
210 | unsigned long *hpa, enum dma_data_direction *direction); | |
d905c5df | 211 | #else |
b348aa65 | 212 | static inline void iommu_register_group(struct iommu_table_group *table_group, |
d905c5df AK |
213 | int pci_domain_number, |
214 | unsigned long pe_num) | |
215 | { | |
216 | } | |
217 | ||
218 | static inline int iommu_add_device(struct device *dev) | |
219 | { | |
220 | return 0; | |
221 | } | |
222 | ||
223 | static inline void iommu_del_device(struct device *dev) | |
224 | { | |
225 | } | |
4ad04e59 NA |
226 | |
227 | static inline int __init tce_iommu_bus_notifier_init(void) | |
228 | { | |
229 | return 0; | |
230 | } | |
d905c5df AK |
231 | #endif /* !CONFIG_IOMMU_API */ |
232 | ||
2db4928b BH |
233 | #else |
234 | ||
235 | static inline void *get_iommu_table_base(struct device *dev) | |
236 | { | |
237 | return NULL; | |
238 | } | |
239 | ||
240 | static inline int dma_iommu_dma_supported(struct device *dev, u64 mask) | |
241 | { | |
242 | return 0; | |
243 | } | |
244 | ||
245 | #endif /* CONFIG_PPC64 */ | |
246 | ||
0690cbd2 JR |
247 | extern int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, |
248 | struct scatterlist *sglist, int nelems, | |
249 | unsigned long mask, | |
250 | enum dma_data_direction direction, | |
251 | struct dma_attrs *attrs); | |
252 | extern void ppc_iommu_unmap_sg(struct iommu_table *tbl, | |
253 | struct scatterlist *sglist, | |
254 | int nelems, | |
255 | enum dma_data_direction direction, | |
256 | struct dma_attrs *attrs); | |
9b6b563c | 257 | |
fb3475e9 FT |
258 | extern void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl, |
259 | size_t size, dma_addr_t *dma_handle, | |
260 | unsigned long mask, gfp_t flag, int node); | |
9b6b563c | 261 | extern void iommu_free_coherent(struct iommu_table *tbl, size_t size, |
12d04eef | 262 | void *vaddr, dma_addr_t dma_handle); |
f9226d57 MN |
263 | extern dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, |
264 | struct page *page, unsigned long offset, | |
265 | size_t size, unsigned long mask, | |
266 | enum dma_data_direction direction, | |
267 | struct dma_attrs *attrs); | |
268 | extern void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle, | |
269 | size_t size, enum dma_data_direction direction, | |
270 | struct dma_attrs *attrs); | |
9b6b563c PM |
271 | |
272 | extern void iommu_init_early_pSeries(void); | |
798248a3 | 273 | extern void iommu_init_early_dart(struct pci_controller_ops *controller_ops); |
31c56d82 | 274 | extern void iommu_init_early_pasemi(void); |
9b6b563c | 275 | |
1beb6a7d | 276 | extern void alloc_dart_table(void); |
7e11580b JB |
277 | #if defined(CONFIG_PPC64) && defined(CONFIG_PM) |
278 | static inline void iommu_save(void) | |
279 | { | |
280 | if (ppc_md.iommu_save) | |
281 | ppc_md.iommu_save(); | |
282 | } | |
283 | ||
284 | static inline void iommu_restore(void) | |
285 | { | |
286 | if (ppc_md.iommu_restore) | |
287 | ppc_md.iommu_restore(); | |
288 | } | |
289 | #endif | |
9b6b563c | 290 | |
4e13c1ac AK |
291 | /* The API to support IOMMU operations for VFIO */ |
292 | extern int iommu_tce_clear_param_check(struct iommu_table *tbl, | |
293 | unsigned long ioba, unsigned long tce_value, | |
294 | unsigned long npages); | |
295 | extern int iommu_tce_put_param_check(struct iommu_table *tbl, | |
296 | unsigned long ioba, unsigned long tce); | |
4e13c1ac AK |
297 | |
298 | extern void iommu_flush_tce(struct iommu_table *tbl); | |
299 | extern int iommu_take_ownership(struct iommu_table *tbl); | |
300 | extern void iommu_release_ownership(struct iommu_table *tbl); | |
301 | ||
302 | extern enum dma_data_direction iommu_tce_direction(unsigned long tce); | |
10b35b2b | 303 | extern unsigned long iommu_direction_to_tce_perm(enum dma_data_direction dir); |
4e13c1ac | 304 | |
88ced031 | 305 | #endif /* __KERNEL__ */ |
9b6b563c | 306 | #endif /* _ASM_IOMMU_H */ |