projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
AMD IOMMU: some set_device_domain cleanups
[deliverable/linux.git] / arch / x86 / kernel / amd_iommu.c
1 /*
2 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/pci.h>
21 #include <linux/gfp.h>
22 #include <linux/bitops.h>
23 #include <linux/scatterlist.h>
24 #include <linux/iommu-helper.h>
25 #include <asm/proto.h>
26 #include <asm/iommu.h>
27 #include <asm/amd_iommu_types.h>
28 #include <asm/amd_iommu.h>
29
30 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
31
32 #define EXIT_LOOP_COUNT 10000000
33
34 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
35
36 /* A list of preallocated protection domains */
37 static LIST_HEAD(iommu_pd_list);
38 static DEFINE_SPINLOCK(iommu_pd_list_lock);
39
40 /*
41 * general struct to manage commands send to an IOMMU
42 */
43 struct iommu_cmd {
44 u32 data[4];
45 };
46
47 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
48 struct unity_map_entry *e);
49
50 /* returns !0 if the IOMMU is caching non-present entries in its TLB */
51 static int iommu_has_npcache(struct amd_iommu *iommu)
52 {
53 return iommu->cap & IOMMU_CAP_NPCACHE;
54 }
55
56 /****************************************************************************
57 *
58 * Interrupt handling functions
59 *
60 ****************************************************************************/
61
62 static void iommu_print_event(void *__evt)
63 {
64 u32 *event = __evt;
65 int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
66 int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
67 int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
68 int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
69 u64 address = (u64)(((u64)event[3]) << 32) | event[2];
70
71 printk(KERN_ERR "AMD IOMMU: Event logged [");
72
73 switch (type) {
74 case EVENT_TYPE_ILL_DEV:
75 printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
76 "address=0x%016llx flags=0x%04x]\n",
77 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
78 address, flags);
79 break;
80 case EVENT_TYPE_IO_FAULT:
81 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
82 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
83 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
84 domid, address, flags);
85 break;
86 case EVENT_TYPE_DEV_TAB_ERR:
87 printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
88 "address=0x%016llx flags=0x%04x]\n",
89 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
90 address, flags);
91 break;
92 case EVENT_TYPE_PAGE_TAB_ERR:
93 printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
94 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
95 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
96 domid, address, flags);
97 break;
98 case EVENT_TYPE_ILL_CMD:
99 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
100 break;
101 case EVENT_TYPE_CMD_HARD_ERR:
102 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
103 "flags=0x%04x]\n", address, flags);
104 break;
105 case EVENT_TYPE_IOTLB_INV_TO:
106 printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
107 "address=0x%016llx]\n",
108 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
109 address);
110 break;
111 case EVENT_TYPE_INV_DEV_REQ:
112 printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
113 "address=0x%016llx flags=0x%04x]\n",
114 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
115 address, flags);
116 break;
117 default:
118 printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
119 }
120 }
121
122 static void iommu_poll_events(struct amd_iommu *iommu)
123 {
124 u32 head, tail;
125 unsigned long flags;
126
127 spin_lock_irqsave(&iommu->lock, flags);
128
129 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
130 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
131
132 while (head != tail) {
133 iommu_print_event(iommu->evt_buf + head);
134 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
135 }
136
137 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
138
139 spin_unlock_irqrestore(&iommu->lock, flags);
140 }
141
142 irqreturn_t amd_iommu_int_handler(int irq, void *data)
143 {
144 struct amd_iommu *iommu;
145
146 list_for_each_entry(iommu, &amd_iommu_list, list)
147 iommu_poll_events(iommu);
148
149 return IRQ_HANDLED;
150 }
151
152 /****************************************************************************
153 *
154 * IOMMU command queuing functions
155 *
156 ****************************************************************************/
157
158 /*
159 * Writes the command to the IOMMUs command buffer and informs the
160 * hardware about the new command. Must be called with iommu->lock held.
161 */
162 static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
163 {
164 u32 tail, head;
165 u8 *target;
166
167 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
168 target = iommu->cmd_buf + tail;
169 memcpy_toio(target, cmd, sizeof(*cmd));
170 tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
171 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
172 if (tail == head)
173 return -ENOMEM;
174 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
175
176 return 0;
177 }
178
179 /*
180 * General queuing function for commands. Takes iommu->lock and calls
181 * __iommu_queue_command().
182 */
183 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
184 {
185 unsigned long flags;
186 int ret;
187
188 spin_lock_irqsave(&iommu->lock, flags);
189 ret = __iommu_queue_command(iommu, cmd);
190 spin_unlock_irqrestore(&iommu->lock, flags);
191
192 return ret;
193 }
194
195 /*
196 * This function is called whenever we need to ensure that the IOMMU has
197 * completed execution of all commands we sent. It sends a
198 * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
199 * us about that by writing a value to a physical address we pass with
200 * the command.
201 */
202 static int iommu_completion_wait(struct amd_iommu *iommu)
203 {
204 int ret, ready = 0;
205 unsigned status = 0;
206 struct iommu_cmd cmd;
207 unsigned long i = 0;
208
209 memset(&cmd, 0, sizeof(cmd));
210 cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
211 CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
212
213 iommu->need_sync = 0;
214
215 ret = iommu_queue_command(iommu, &cmd);
216
217 if (ret)
218 return ret;
219
220 while (!ready && (i < EXIT_LOOP_COUNT)) {
221 ++i;
222 /* wait for the bit to become one */
223 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
224 ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
225 }
226
227 /* set bit back to zero */
228 status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
229 writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
230
231 if (unlikely((i == EXIT_LOOP_COUNT) && printk_ratelimit()))
232 printk(KERN_WARNING "AMD IOMMU: Completion wait loop failed\n");
233
234 return 0;
235 }
236
237 /*
238 * Command send function for invalidating a device table entry
239 */
240 static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
241 {
242 struct iommu_cmd cmd;
243
244 BUG_ON(iommu == NULL);
245
246 memset(&cmd, 0, sizeof(cmd));
247 CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
248 cmd.data[0] = devid;
249
250 iommu->need_sync = 1;
251
252 return iommu_queue_command(iommu, &cmd);
253 }
254
255 /*
256 * Generic command send function for invalidaing TLB entries
257 */
258 static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
259 u64 address, u16 domid, int pde, int s)
260 {
261 struct iommu_cmd cmd;
262
263 memset(&cmd, 0, sizeof(cmd));
264 address &= PAGE_MASK;
265 CMD_SET_TYPE(&cmd, CMD_INV_IOMMU_PAGES);
266 cmd.data[1] |= domid;
267 cmd.data[2] = lower_32_bits(address);
268 cmd.data[3] = upper_32_bits(address);
269 if (s) /* size bit - we flush more than one 4kb page */
270 cmd.data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
271 if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
272 cmd.data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
273
274 iommu->need_sync = 1;
275
276 return iommu_queue_command(iommu, &cmd);
277 }
278
279 /*
280 * TLB invalidation function which is called from the mapping functions.
281 * It invalidates a single PTE if the range to flush is within a single
282 * page. Otherwise it flushes the whole TLB of the IOMMU.
283 */
284 static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
285 u64 address, size_t size)
286 {
287 int s = 0;
288 unsigned pages = iommu_num_pages(address, size);
289
290 address &= PAGE_MASK;
291
292 if (pages > 1) {
293 /*
294 * If we have to flush more than one page, flush all
295 * TLB entries for this domain
296 */
297 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
298 s = 1;
299 }
300
301 iommu_queue_inv_iommu_pages(iommu, address, domid, 0, s);
302
303 return 0;
304 }
305
306 /* Flush the whole IO/TLB for a given protection domain */
307 static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
308 {
309 u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
310
311 iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1);
312 }
313
314 /****************************************************************************
315 *
316 * The functions below are used the create the page table mappings for
317 * unity mapped regions.
318 *
319 ****************************************************************************/
320
321 /*
322 * Generic mapping functions. It maps a physical address into a DMA
323 * address space. It allocates the page table pages if necessary.
324 * In the future it can be extended to a generic mapping function
325 * supporting all features of AMD IOMMU page tables like level skipping
326 * and full 64 bit address spaces.
327 */
328 static int iommu_map(struct protection_domain *dom,
329 unsigned long bus_addr,
330 unsigned long phys_addr,
331 int prot)
332 {
333 u64 __pte, *pte, *page;
334
335 bus_addr = PAGE_ALIGN(bus_addr);
336 phys_addr = PAGE_ALIGN(bus_addr);
337
338 /* only support 512GB address spaces for now */
339 if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))
340 return -EINVAL;
341
342 pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];
343
344 if (!IOMMU_PTE_PRESENT(*pte)) {
345 page = (u64 *)get_zeroed_page(GFP_KERNEL);
346 if (!page)
347 return -ENOMEM;
348 *pte = IOMMU_L2_PDE(virt_to_phys(page));
349 }
350
351 pte = IOMMU_PTE_PAGE(*pte);
352 pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
353
354 if (!IOMMU_PTE_PRESENT(*pte)) {
355 page = (u64 *)get_zeroed_page(GFP_KERNEL);
356 if (!page)
357 return -ENOMEM;
358 *pte = IOMMU_L1_PDE(virt_to_phys(page));
359 }
360
361 pte = IOMMU_PTE_PAGE(*pte);
362 pte = &pte[IOMMU_PTE_L0_INDEX(bus_addr)];
363
364 if (IOMMU_PTE_PRESENT(*pte))
365 return -EBUSY;
366
367 __pte = phys_addr | IOMMU_PTE_P;
368 if (prot & IOMMU_PROT_IR)
369 __pte |= IOMMU_PTE_IR;
370 if (prot & IOMMU_PROT_IW)
371 __pte |= IOMMU_PTE_IW;
372
373 *pte = __pte;
374
375 return 0;
376 }
377
378 /*
379 * This function checks if a specific unity mapping entry is needed for
380 * this specific IOMMU.
381 */
382 static int iommu_for_unity_map(struct amd_iommu *iommu,
383 struct unity_map_entry *entry)
384 {
385 u16 bdf, i;
386
387 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
388 bdf = amd_iommu_alias_table[i];
389 if (amd_iommu_rlookup_table[bdf] == iommu)
390 return 1;
391 }
392
393 return 0;
394 }
395
396 /*
397 * Init the unity mappings for a specific IOMMU in the system
398 *
399 * Basically iterates over all unity mapping entries and applies them to
400 * the default domain DMA of that IOMMU if necessary.
401 */
402 static int iommu_init_unity_mappings(struct amd_iommu *iommu)
403 {
404 struct unity_map_entry *entry;
405 int ret;
406
407 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
408 if (!iommu_for_unity_map(iommu, entry))
409 continue;
410 ret = dma_ops_unity_map(iommu->default_dom, entry);
411 if (ret)
412 return ret;
413 }
414
415 return 0;
416 }
417
418 /*
419 * This function actually applies the mapping to the page table of the
420 * dma_ops domain.
421 */
422 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
423 struct unity_map_entry *e)
424 {
425 u64 addr;
426 int ret;
427
428 for (addr = e->address_start; addr < e->address_end;
429 addr += PAGE_SIZE) {
430 ret = iommu_map(&dma_dom->domain, addr, addr, e->prot);
431 if (ret)
432 return ret;
433 /*
434 * if unity mapping is in aperture range mark the page
435 * as allocated in the aperture
436 */
437 if (addr < dma_dom->aperture_size)
438 __set_bit(addr >> PAGE_SHIFT, dma_dom->bitmap);
439 }
440
441 return 0;
442 }
443
444 /*
445 * Inits the unity mappings required for a specific device
446 */
447 static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
448 u16 devid)
449 {
450 struct unity_map_entry *e;
451 int ret;
452
453 list_for_each_entry(e, &amd_iommu_unity_map, list) {
454 if (!(devid >= e->devid_start && devid <= e->devid_end))
455 continue;
456 ret = dma_ops_unity_map(dma_dom, e);
457 if (ret)
458 return ret;
459 }
460
461 return 0;
462 }
463
464 /****************************************************************************
465 *
466 * The next functions belong to the address allocator for the dma_ops
467 * interface functions. They work like the allocators in the other IOMMU
468 * drivers. Its basically a bitmap which marks the allocated pages in
469 * the aperture. Maybe it could be enhanced in the future to a more
470 * efficient allocator.
471 *
472 ****************************************************************************/
473 static unsigned long dma_mask_to_pages(unsigned long mask)
474 {
475 return (mask >> PAGE_SHIFT) +
476 (PAGE_ALIGN(mask & ~PAGE_MASK) >> PAGE_SHIFT);
477 }
478
479 /*
480 * The address allocator core function.
481 *
482 * called with domain->lock held
483 */
484 static unsigned long dma_ops_alloc_addresses(struct device *dev,
485 struct dma_ops_domain *dom,
486 unsigned int pages,
487 unsigned long align_mask)
488 {
489 unsigned long limit = dma_mask_to_pages(*dev->dma_mask);
490 unsigned long address;
491 unsigned long size = dom->aperture_size >> PAGE_SHIFT;
492 unsigned long boundary_size;
493
494 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
495 PAGE_SIZE) >> PAGE_SHIFT;
496 limit = limit < size ? limit : size;
497
498 if (dom->next_bit >= limit) {
499 dom->next_bit = 0;
500 dom->need_flush = true;
501 }
502
503 address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,
504 0 , boundary_size, align_mask);
505 if (address == -1) {
506 address = iommu_area_alloc(dom->bitmap, limit, 0, pages,
507 0, boundary_size, align_mask);
508 dom->need_flush = true;
509 }
510
511 if (likely(address != -1)) {
512 dom->next_bit = address + pages;
513 address <<= PAGE_SHIFT;
514 } else
515 address = bad_dma_address;
516
517 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
518
519 return address;
520 }
521
522 /*
523 * The address free function.
524 *
525 * called with domain->lock held
526 */
527 static void dma_ops_free_addresses(struct dma_ops_domain *dom,
528 unsigned long address,
529 unsigned int pages)
530 {
531 address >>= PAGE_SHIFT;
532 iommu_area_free(dom->bitmap, address, pages);
533 }
534
535 /****************************************************************************
536 *
537 * The next functions belong to the domain allocation. A domain is
538 * allocated for every IOMMU as the default domain. If device isolation
539 * is enabled, every device get its own domain. The most important thing
540 * about domains is the page table mapping the DMA address space they
541 * contain.
542 *
543 ****************************************************************************/
544
545 static u16 domain_id_alloc(void)
546 {
547 unsigned long flags;
548 int id;
549
550 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
551 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
552 BUG_ON(id == 0);
553 if (id > 0 && id < MAX_DOMAIN_ID)
554 __set_bit(id, amd_iommu_pd_alloc_bitmap);
555 else
556 id = 0;
557 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
558
559 return id;
560 }
561
562 /*
563 * Used to reserve address ranges in the aperture (e.g. for exclusion
564 * ranges.
565 */
566 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
567 unsigned long start_page,
568 unsigned int pages)
569 {
570 unsigned int last_page = dom->aperture_size >> PAGE_SHIFT;
571
572 if (start_page + pages > last_page)
573 pages = last_page - start_page;
574
575 set_bit_string(dom->bitmap, start_page, pages);
576 }
577
578 static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)
579 {
580 int i, j;
581 u64 *p1, *p2, *p3;
582
583 p1 = dma_dom->domain.pt_root;
584
585 if (!p1)
586 return;
587
588 for (i = 0; i < 512; ++i) {
589 if (!IOMMU_PTE_PRESENT(p1[i]))
590 continue;
591
592 p2 = IOMMU_PTE_PAGE(p1[i]);
593 for (j = 0; j < 512; ++i) {
594 if (!IOMMU_PTE_PRESENT(p2[j]))
595 continue;
596 p3 = IOMMU_PTE_PAGE(p2[j]);
597 free_page((unsigned long)p3);
598 }
599
600 free_page((unsigned long)p2);
601 }
602
603 free_page((unsigned long)p1);
604 }
605
606 /*
607 * Free a domain, only used if something went wrong in the
608 * allocation path and we need to free an already allocated page table
609 */
610 static void dma_ops_domain_free(struct dma_ops_domain *dom)
611 {
612 if (!dom)
613 return;
614
615 dma_ops_free_pagetable(dom);
616
617 kfree(dom->pte_pages);
618
619 kfree(dom->bitmap);
620
621 kfree(dom);
622 }
623
624 /*
625 * Allocates a new protection domain usable for the dma_ops functions.
626 * It also intializes the page table and the address allocator data
627 * structures required for the dma_ops interface
628 */
629 static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
630 unsigned order)
631 {
632 struct dma_ops_domain *dma_dom;
633 unsigned i, num_pte_pages;
634 u64 *l2_pde;
635 u64 address;
636
637 /*
638 * Currently the DMA aperture must be between 32 MB and 1GB in size
639 */
640 if ((order < 25) || (order > 30))
641 return NULL;
642
643 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
644 if (!dma_dom)
645 return NULL;
646
647 spin_lock_init(&dma_dom->domain.lock);
648
649 dma_dom->domain.id = domain_id_alloc();
650 if (dma_dom->domain.id == 0)
651 goto free_dma_dom;
652 dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
653 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
654 dma_dom->domain.priv = dma_dom;
655 if (!dma_dom->domain.pt_root)
656 goto free_dma_dom;
657 dma_dom->aperture_size = (1ULL << order);
658 dma_dom->bitmap = kzalloc(dma_dom->aperture_size / (PAGE_SIZE * 8),
659 GFP_KERNEL);
660 if (!dma_dom->bitmap)
661 goto free_dma_dom;
662 /*
663 * mark the first page as allocated so we never return 0 as
664 * a valid dma-address. So we can use 0 as error value
665 */
666 dma_dom->bitmap[0] = 1;
667 dma_dom->next_bit = 0;
668
669 dma_dom->need_flush = false;
670 dma_dom->target_dev = 0xffff;
671
672 /* Intialize the exclusion range if necessary */
673 if (iommu->exclusion_start &&
674 iommu->exclusion_start < dma_dom->aperture_size) {
675 unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
676 int pages = iommu_num_pages(iommu->exclusion_start,
677 iommu->exclusion_length);
678 dma_ops_reserve_addresses(dma_dom, startpage, pages);
679 }
680
681 /*
682 * At the last step, build the page tables so we don't need to
683 * allocate page table pages in the dma_ops mapping/unmapping
684 * path.
685 */
686 num_pte_pages = dma_dom->aperture_size / (PAGE_SIZE * 512);
687 dma_dom->pte_pages = kzalloc(num_pte_pages * sizeof(void *),
688 GFP_KERNEL);
689 if (!dma_dom->pte_pages)
690 goto free_dma_dom;
691
692 l2_pde = (u64 *)get_zeroed_page(GFP_KERNEL);
693 if (l2_pde == NULL)
694 goto free_dma_dom;
695
696 dma_dom->domain.pt_root[0] = IOMMU_L2_PDE(virt_to_phys(l2_pde));
697
698 for (i = 0; i < num_pte_pages; ++i) {
699 dma_dom->pte_pages[i] = (u64 *)get_zeroed_page(GFP_KERNEL);
700 if (!dma_dom->pte_pages[i])
701 goto free_dma_dom;
702 address = virt_to_phys(dma_dom->pte_pages[i]);
703 l2_pde[i] = IOMMU_L1_PDE(address);
704 }
705
706 return dma_dom;
707
708 free_dma_dom:
709 dma_ops_domain_free(dma_dom);
710
711 return NULL;
712 }
713
714 /*
715 * Find out the protection domain structure for a given PCI device. This
716 * will give us the pointer to the page table root for example.
717 */
718 static struct protection_domain *domain_for_device(u16 devid)
719 {
720 struct protection_domain *dom;
721 unsigned long flags;
722
723 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
724 dom = amd_iommu_pd_table[devid];
725 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
726
727 return dom;
728 }
729
730 /*
731 * If a device is not yet associated with a domain, this function does
732 * assigns it visible for the hardware
733 */
734 static void set_device_domain(struct amd_iommu *iommu,
735 struct protection_domain *domain,
736 u16 devid)
737 {
738 unsigned long flags;
739
740 u64 pte_root = virt_to_phys(domain->pt_root);
741
742 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
743 << DEV_ENTRY_MODE_SHIFT;
744 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
745
746 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
747 amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
748 amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
749 amd_iommu_dev_table[devid].data[2] = domain->id;
750
751 amd_iommu_pd_table[devid] = domain;
752 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
753
754 iommu_queue_inv_dev_entry(iommu, devid);
755
756 iommu->need_sync = 1;
757 }
758
759 /*****************************************************************************
760 *
761 * The next functions belong to the dma_ops mapping/unmapping code.
762 *
763 *****************************************************************************/
764
765 /*
766 * This function checks if the driver got a valid device from the caller to
767 * avoid dereferencing invalid pointers.
768 */
769 static bool check_device(struct device *dev)
770 {
771 if (!dev || !dev->dma_mask)
772 return false;
773
774 return true;
775 }
776
777 /*
778 * In this function the list of preallocated protection domains is traversed to
779 * find the domain for a specific device
780 */
781 static struct dma_ops_domain *find_protection_domain(u16 devid)
782 {
783 struct dma_ops_domain *entry, *ret = NULL;
784 unsigned long flags;
785
786 if (list_empty(&iommu_pd_list))
787 return NULL;
788
789 spin_lock_irqsave(&iommu_pd_list_lock, flags);
790
791 list_for_each_entry(entry, &iommu_pd_list, list) {
792 if (entry->target_dev == devid) {
793 ret = entry;
794 list_del(&ret->list);
795 break;
796 }
797 }
798
799 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
800
801 return ret;
802 }
803
804 /*
805 * In the dma_ops path we only have the struct device. This function
806 * finds the corresponding IOMMU, the protection domain and the
807 * requestor id for a given device.
808 * If the device is not yet associated with a domain this is also done
809 * in this function.
810 */
811 static int get_device_resources(struct device *dev,
812 struct amd_iommu **iommu,
813 struct protection_domain **domain,
814 u16 *bdf)
815 {
816 struct dma_ops_domain *dma_dom;
817 struct pci_dev *pcidev;
818 u16 _bdf;
819
820 *iommu = NULL;
821 *domain = NULL;
822 *bdf = 0xffff;
823
824 if (dev->bus != &pci_bus_type)
825 return 0;
826
827 pcidev = to_pci_dev(dev);
828 _bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
829
830 /* device not translated by any IOMMU in the system? */
831 if (_bdf > amd_iommu_last_bdf)
832 return 0;
833
834 *bdf = amd_iommu_alias_table[_bdf];
835
836 *iommu = amd_iommu_rlookup_table[*bdf];
837 if (*iommu == NULL)
838 return 0;
839 *domain = domain_for_device(*bdf);
840 if (*domain == NULL) {
841 dma_dom = find_protection_domain(*bdf);
842 if (!dma_dom)
843 dma_dom = (*iommu)->default_dom;
844 *domain = &dma_dom->domain;
845 set_device_domain(*iommu, *domain, *bdf);
846 printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
847 "device ", (*domain)->id);
848 print_devid(_bdf, 1);
849 }
850
851 return 1;
852 }
853
854 /*
855 * This is the generic map function. It maps one 4kb page at paddr to
856 * the given address in the DMA address space for the domain.
857 */
858 static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu,
859 struct dma_ops_domain *dom,
860 unsigned long address,
861 phys_addr_t paddr,
862 int direction)
863 {
864 u64 *pte, __pte;
865
866 WARN_ON(address > dom->aperture_size);
867
868 paddr &= PAGE_MASK;
869
870 pte = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
871 pte += IOMMU_PTE_L0_INDEX(address);
872
873 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
874
875 if (direction == DMA_TO_DEVICE)
876 __pte |= IOMMU_PTE_IR;
877 else if (direction == DMA_FROM_DEVICE)
878 __pte |= IOMMU_PTE_IW;
879 else if (direction == DMA_BIDIRECTIONAL)
880 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
881
882 WARN_ON(*pte);
883
884 *pte = __pte;
885
886 return (dma_addr_t)address;
887 }
888
889 /*
890 * The generic unmapping function for on page in the DMA address space.
891 */
892 static void dma_ops_domain_unmap(struct amd_iommu *iommu,
893 struct dma_ops_domain *dom,
894 unsigned long address)
895 {
896 u64 *pte;
897
898 if (address >= dom->aperture_size)
899 return;
900
901 WARN_ON(address & 0xfffULL || address > dom->aperture_size);
902
903 pte = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
904 pte += IOMMU_PTE_L0_INDEX(address);
905
906 WARN_ON(!*pte);
907
908 *pte = 0ULL;
909 }
910
911 /*
912 * This function contains common code for mapping of a physically
913 * contiguous memory region into DMA address space. It is uses by all
914 * mapping functions provided by this IOMMU driver.
915 * Must be called with the domain lock held.
916 */
917 static dma_addr_t __map_single(struct device *dev,
918 struct amd_iommu *iommu,
919 struct dma_ops_domain *dma_dom,
920 phys_addr_t paddr,
921 size_t size,
922 int dir,
923 bool align)
924 {
925 dma_addr_t offset = paddr & ~PAGE_MASK;
926 dma_addr_t address, start;
927 unsigned int pages;
928 unsigned long align_mask = 0;
929 int i;
930
931 pages = iommu_num_pages(paddr, size);
932 paddr &= PAGE_MASK;
933
934 if (align)
935 align_mask = (1UL << get_order(size)) - 1;
936
937 address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask);
938 if (unlikely(address == bad_dma_address))
939 goto out;
940
941 start = address;
942 for (i = 0; i < pages; ++i) {
943 dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
944 paddr += PAGE_SIZE;
945 start += PAGE_SIZE;
946 }
947 address += offset;
948
949 if (unlikely(dma_dom->need_flush && !iommu_fullflush)) {
950 iommu_flush_tlb(iommu, dma_dom->domain.id);
951 dma_dom->need_flush = false;
952 } else if (unlikely(iommu_has_npcache(iommu)))
953 iommu_flush_pages(iommu, dma_dom->domain.id, address, size);
954
955 out:
956 return address;
957 }
958
959 /*
960 * Does the reverse of the __map_single function. Must be called with
961 * the domain lock held too
962 */
963 static void __unmap_single(struct amd_iommu *iommu,
964 struct dma_ops_domain *dma_dom,
965 dma_addr_t dma_addr,
966 size_t size,
967 int dir)
968 {
969 dma_addr_t i, start;
970 unsigned int pages;
971
972 if ((dma_addr == 0) || (dma_addr + size > dma_dom->aperture_size))
973 return;
974
975 pages = iommu_num_pages(dma_addr, size);
976 dma_addr &= PAGE_MASK;
977 start = dma_addr;
978
979 for (i = 0; i < pages; ++i) {
980 dma_ops_domain_unmap(iommu, dma_dom, start);
981 start += PAGE_SIZE;
982 }
983
984 dma_ops_free_addresses(dma_dom, dma_addr, pages);
985
986 if (iommu_fullflush)
987 iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
988 }
989
990 /*
991 * The exported map_single function for dma_ops.
992 */
993 static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
994 size_t size, int dir)
995 {
996 unsigned long flags;
997 struct amd_iommu *iommu;
998 struct protection_domain *domain;
999 u16 devid;
1000 dma_addr_t addr;
1001
1002 if (!check_device(dev))
1003 return bad_dma_address;
1004
1005 get_device_resources(dev, &iommu, &domain, &devid);
1006
1007 if (iommu == NULL || domain == NULL)
1008 /* device not handled by any AMD IOMMU */
1009 return (dma_addr_t)paddr;
1010
1011 spin_lock_irqsave(&domain->lock, flags);
1012 addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false);
1013 if (addr == bad_dma_address)
1014 goto out;
1015
1016 if (unlikely(iommu->need_sync))
1017 iommu_completion_wait(iommu);
1018
1019 out:
1020 spin_unlock_irqrestore(&domain->lock, flags);
1021
1022 return addr;
1023 }
1024
1025 /*
1026 * The exported unmap_single function for dma_ops.
1027 */
1028 static void unmap_single(struct device *dev, dma_addr_t dma_addr,
1029 size_t size, int dir)
1030 {
1031 unsigned long flags;
1032 struct amd_iommu *iommu;
1033 struct protection_domain *domain;
1034 u16 devid;
1035
1036 if (!check_device(dev) ||
1037 !get_device_resources(dev, &iommu, &domain, &devid))
1038 /* device not handled by any AMD IOMMU */
1039 return;
1040
1041 spin_lock_irqsave(&domain->lock, flags);
1042
1043 __unmap_single(iommu, domain->priv, dma_addr, size, dir);
1044
1045 if (unlikely(iommu->need_sync))
1046 iommu_completion_wait(iommu);
1047
1048 spin_unlock_irqrestore(&domain->lock, flags);
1049 }
1050
1051 /*
1052 * This is a special map_sg function which is used if we should map a
1053 * device which is not handled by an AMD IOMMU in the system.
1054 */
1055 static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
1056 int nelems, int dir)
1057 {
1058 struct scatterlist *s;
1059 int i;
1060
1061 for_each_sg(sglist, s, nelems, i) {
1062 s->dma_address = (dma_addr_t)sg_phys(s);
1063 s->dma_length = s->length;
1064 }
1065
1066 return nelems;
1067 }
1068
1069 /*
1070 * The exported map_sg function for dma_ops (handles scatter-gather
1071 * lists).
1072 */
1073 static int map_sg(struct device *dev, struct scatterlist *sglist,
1074 int nelems, int dir)
1075 {
1076 unsigned long flags;
1077 struct amd_iommu *iommu;
1078 struct protection_domain *domain;
1079 u16 devid;
1080 int i;
1081 struct scatterlist *s;
1082 phys_addr_t paddr;
1083 int mapped_elems = 0;
1084
1085 if (!check_device(dev))
1086 return 0;
1087
1088 get_device_resources(dev, &iommu, &domain, &devid);
1089
1090 if (!iommu || !domain)
1091 return map_sg_no_iommu(dev, sglist, nelems, dir);
1092
1093 spin_lock_irqsave(&domain->lock, flags);
1094
1095 for_each_sg(sglist, s, nelems, i) {
1096 paddr = sg_phys(s);
1097
1098 s->dma_address = __map_single(dev, iommu, domain->priv,
1099 paddr, s->length, dir, false);
1100
1101 if (s->dma_address) {
1102 s->dma_length = s->length;
1103 mapped_elems++;
1104 } else
1105 goto unmap;
1106 }
1107
1108 if (unlikely(iommu->need_sync))
1109 iommu_completion_wait(iommu);
1110
1111 out:
1112 spin_unlock_irqrestore(&domain->lock, flags);
1113
1114 return mapped_elems;
1115 unmap:
1116 for_each_sg(sglist, s, mapped_elems, i) {
1117 if (s->dma_address)
1118 __unmap_single(iommu, domain->priv, s->dma_address,
1119 s->dma_length, dir);
1120 s->dma_address = s->dma_length = 0;
1121 }
1122
1123 mapped_elems = 0;
1124
1125 goto out;
1126 }
1127
1128 /*
1129 * The exported map_sg function for dma_ops (handles scatter-gather
1130 * lists).
1131 */
1132 static void unmap_sg(struct device *dev, struct scatterlist *sglist,
1133 int nelems, int dir)
1134 {
1135 unsigned long flags;
1136 struct amd_iommu *iommu;
1137 struct protection_domain *domain;
1138 struct scatterlist *s;
1139 u16 devid;
1140 int i;
1141
1142 if (!check_device(dev) ||
1143 !get_device_resources(dev, &iommu, &domain, &devid))
1144 return;
1145
1146 spin_lock_irqsave(&domain->lock, flags);
1147
1148 for_each_sg(sglist, s, nelems, i) {
1149 __unmap_single(iommu, domain->priv, s->dma_address,
1150 s->dma_length, dir);
1151 s->dma_address = s->dma_length = 0;
1152 }
1153
1154 if (unlikely(iommu->need_sync))
1155 iommu_completion_wait(iommu);
1156
1157 spin_unlock_irqrestore(&domain->lock, flags);
1158 }
1159
1160 /*
1161 * The exported alloc_coherent function for dma_ops.
1162 */
1163 static void *alloc_coherent(struct device *dev, size_t size,
1164 dma_addr_t *dma_addr, gfp_t flag)
1165 {
1166 unsigned long flags;
1167 void *virt_addr;
1168 struct amd_iommu *iommu;
1169 struct protection_domain *domain;
1170 u16 devid;
1171 phys_addr_t paddr;
1172
1173 if (!check_device(dev))
1174 return NULL;
1175
1176 virt_addr = (void *)__get_free_pages(flag, get_order(size));
1177 if (!virt_addr)
1178 return 0;
1179
1180 memset(virt_addr, 0, size);
1181 paddr = virt_to_phys(virt_addr);
1182
1183 get_device_resources(dev, &iommu, &domain, &devid);
1184
1185 if (!iommu || !domain) {
1186 *dma_addr = (dma_addr_t)paddr;
1187 return virt_addr;
1188 }
1189
1190 spin_lock_irqsave(&domain->lock, flags);
1191
1192 *dma_addr = __map_single(dev, iommu, domain->priv, paddr,
1193 size, DMA_BIDIRECTIONAL, true);
1194
1195 if (*dma_addr == bad_dma_address) {
1196 free_pages((unsigned long)virt_addr, get_order(size));
1197 virt_addr = NULL;
1198 goto out;
1199 }
1200
1201 if (unlikely(iommu->need_sync))
1202 iommu_completion_wait(iommu);
1203
1204 out:
1205 spin_unlock_irqrestore(&domain->lock, flags);
1206
1207 return virt_addr;
1208 }
1209
1210 /*
1211 * The exported free_coherent function for dma_ops.
1212 */
1213 static void free_coherent(struct device *dev, size_t size,
1214 void *virt_addr, dma_addr_t dma_addr)
1215 {
1216 unsigned long flags;
1217 struct amd_iommu *iommu;
1218 struct protection_domain *domain;
1219 u16 devid;
1220
1221 if (!check_device(dev))
1222 return;
1223
1224 get_device_resources(dev, &iommu, &domain, &devid);
1225
1226 if (!iommu || !domain)
1227 goto free_mem;
1228
1229 spin_lock_irqsave(&domain->lock, flags);
1230
1231 __unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
1232
1233 if (unlikely(iommu->need_sync))
1234 iommu_completion_wait(iommu);
1235
1236 spin_unlock_irqrestore(&domain->lock, flags);
1237
1238 free_mem:
1239 free_pages((unsigned long)virt_addr, get_order(size));
1240 }
1241
1242 /*
1243 * This function is called by the DMA layer to find out if we can handle a
1244 * particular device. It is part of the dma_ops.
1245 */
1246 static int amd_iommu_dma_supported(struct device *dev, u64 mask)
1247 {
1248 u16 bdf;
1249 struct pci_dev *pcidev;
1250
1251 /* No device or no PCI device */
1252 if (!dev || dev->bus != &pci_bus_type)
1253 return 0;
1254
1255 pcidev = to_pci_dev(dev);
1256
1257 bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
1258
1259 /* Out of our scope? */
1260 if (bdf > amd_iommu_last_bdf)
1261 return 0;
1262
1263 return 1;
1264 }
1265
1266 /*
1267 * The function for pre-allocating protection domains.
1268 *
1269 * If the driver core informs the DMA layer if a driver grabs a device
1270 * we don't need to preallocate the protection domains anymore.
1271 * For now we have to.
1272 */
1273 void prealloc_protection_domains(void)
1274 {
1275 struct pci_dev *dev = NULL;
1276 struct dma_ops_domain *dma_dom;
1277 struct amd_iommu *iommu;
1278 int order = amd_iommu_aperture_order;
1279 u16 devid;
1280
1281 while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
1282 devid = (dev->bus->number << 8) | dev->devfn;
1283 if (devid > amd_iommu_last_bdf)
1284 continue;
1285 devid = amd_iommu_alias_table[devid];
1286 if (domain_for_device(devid))
1287 continue;
1288 iommu = amd_iommu_rlookup_table[devid];
1289 if (!iommu)
1290 continue;
1291 dma_dom = dma_ops_domain_alloc(iommu, order);
1292 if (!dma_dom)
1293 continue;
1294 init_unity_mappings_for_device(dma_dom, devid);
1295 dma_dom->target_dev = devid;
1296
1297 list_add_tail(&dma_dom->list, &iommu_pd_list);
1298 }
1299 }
1300
1301 static struct dma_mapping_ops amd_iommu_dma_ops = {
1302 .alloc_coherent = alloc_coherent,
1303 .free_coherent = free_coherent,
1304 .map_single = map_single,
1305 .unmap_single = unmap_single,
1306 .map_sg = map_sg,
1307 .unmap_sg = unmap_sg,
1308 .dma_supported = amd_iommu_dma_supported,
1309 };
1310
1311 /*
1312 * The function which clues the AMD IOMMU driver into dma_ops.
1313 */
1314 int __init amd_iommu_init_dma_ops(void)
1315 {
1316 struct amd_iommu *iommu;
1317 int order = amd_iommu_aperture_order;
1318 int ret;
1319
1320 /*
1321 * first allocate a default protection domain for every IOMMU we
1322 * found in the system. Devices not assigned to any other
1323 * protection domain will be assigned to the default one.
1324 */
1325 list_for_each_entry(iommu, &amd_iommu_list, list) {
1326 iommu->default_dom = dma_ops_domain_alloc(iommu, order);
1327 if (iommu->default_dom == NULL)
1328 return -ENOMEM;
1329 ret = iommu_init_unity_mappings(iommu);
1330 if (ret)
1331 goto free_domains;
1332 }
1333
1334 /*
1335 * If device isolation is enabled, pre-allocate the protection
1336 * domains for each device.
1337 */
1338 if (amd_iommu_isolate)
1339 prealloc_protection_domains();
1340
1341 iommu_detected = 1;
1342 force_iommu = 1;
1343 bad_dma_address = 0;
1344 #ifdef CONFIG_GART_IOMMU
1345 gart_iommu_aperture_disabled = 1;
1346 gart_iommu_aperture = 0;
1347 #endif
1348
1349 /* Make the driver finally visible to the drivers */
1350 dma_ops = &amd_iommu_dma_ops;
1351
1352 return 0;
1353
1354 free_domains:
1355
1356 list_for_each_entry(iommu, &amd_iommu_list, list) {
1357 if (iommu->default_dom)
1358 dma_ops_domain_free(iommu->default_dom);
1359 }
1360
1361 return ret;
1362 }
Linux kernel - Deliverables
This page took 0.079815 seconds and 5 git commands to generate.