projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
AMD IOMMU: add domain cleanup helper function
[deliverable/linux.git] / arch / x86 / kernel / amd_iommu.c
1 /*
2 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/pci.h>
21 #include <linux/gfp.h>
22 #include <linux/bitops.h>
23 #include <linux/scatterlist.h>
24 #include <linux/iommu-helper.h>
25 #include <asm/proto.h>
26 #include <asm/iommu.h>
27 #include <asm/gart.h>
28 #include <asm/amd_iommu_types.h>
29 #include <asm/amd_iommu.h>
30
31 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
32
33 #define EXIT_LOOP_COUNT 10000000
34
35 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
36
37 /* A list of preallocated protection domains */
38 static LIST_HEAD(iommu_pd_list);
39 static DEFINE_SPINLOCK(iommu_pd_list_lock);
40
41 /*
42 * general struct to manage commands send to an IOMMU
43 */
44 struct iommu_cmd {
45 u32 data[4];
46 };
47
48 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
49 struct unity_map_entry *e);
50 static struct dma_ops_domain *find_protection_domain(u16 devid);
51
52
53 /* returns !0 if the IOMMU is caching non-present entries in its TLB */
54 static int iommu_has_npcache(struct amd_iommu *iommu)
55 {
56 return iommu->cap & (1UL << IOMMU_CAP_NPCACHE);
57 }
58
59 /****************************************************************************
60 *
61 * Interrupt handling functions
62 *
63 ****************************************************************************/
64
65 static void iommu_print_event(void *__evt)
66 {
67 u32 *event = __evt;
68 int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
69 int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
70 int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
71 int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
72 u64 address = (u64)(((u64)event[3]) << 32) | event[2];
73
74 printk(KERN_ERR "AMD IOMMU: Event logged [");
75
76 switch (type) {
77 case EVENT_TYPE_ILL_DEV:
78 printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
79 "address=0x%016llx flags=0x%04x]\n",
80 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
81 address, flags);
82 break;
83 case EVENT_TYPE_IO_FAULT:
84 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
85 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
86 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
87 domid, address, flags);
88 break;
89 case EVENT_TYPE_DEV_TAB_ERR:
90 printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
91 "address=0x%016llx flags=0x%04x]\n",
92 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
93 address, flags);
94 break;
95 case EVENT_TYPE_PAGE_TAB_ERR:
96 printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
97 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
98 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
99 domid, address, flags);
100 break;
101 case EVENT_TYPE_ILL_CMD:
102 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
103 break;
104 case EVENT_TYPE_CMD_HARD_ERR:
105 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
106 "flags=0x%04x]\n", address, flags);
107 break;
108 case EVENT_TYPE_IOTLB_INV_TO:
109 printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
110 "address=0x%016llx]\n",
111 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
112 address);
113 break;
114 case EVENT_TYPE_INV_DEV_REQ:
115 printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
116 "address=0x%016llx flags=0x%04x]\n",
117 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
118 address, flags);
119 break;
120 default:
121 printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
122 }
123 }
124
125 static void iommu_poll_events(struct amd_iommu *iommu)
126 {
127 u32 head, tail;
128 unsigned long flags;
129
130 spin_lock_irqsave(&iommu->lock, flags);
131
132 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
133 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
134
135 while (head != tail) {
136 iommu_print_event(iommu->evt_buf + head);
137 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
138 }
139
140 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
141
142 spin_unlock_irqrestore(&iommu->lock, flags);
143 }
144
145 irqreturn_t amd_iommu_int_handler(int irq, void *data)
146 {
147 struct amd_iommu *iommu;
148
149 list_for_each_entry(iommu, &amd_iommu_list, list)
150 iommu_poll_events(iommu);
151
152 return IRQ_HANDLED;
153 }
154
155 /****************************************************************************
156 *
157 * IOMMU command queuing functions
158 *
159 ****************************************************************************/
160
161 /*
162 * Writes the command to the IOMMUs command buffer and informs the
163 * hardware about the new command. Must be called with iommu->lock held.
164 */
165 static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
166 {
167 u32 tail, head;
168 u8 *target;
169
170 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
171 target = iommu->cmd_buf + tail;
172 memcpy_toio(target, cmd, sizeof(*cmd));
173 tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
174 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
175 if (tail == head)
176 return -ENOMEM;
177 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
178
179 return 0;
180 }
181
182 /*
183 * General queuing function for commands. Takes iommu->lock and calls
184 * __iommu_queue_command().
185 */
186 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
187 {
188 unsigned long flags;
189 int ret;
190
191 spin_lock_irqsave(&iommu->lock, flags);
192 ret = __iommu_queue_command(iommu, cmd);
193 if (!ret)
194 iommu->need_sync = 1;
195 spin_unlock_irqrestore(&iommu->lock, flags);
196
197 return ret;
198 }
199
200 /*
201 * This function waits until an IOMMU has completed a completion
202 * wait command
203 */
204 static void __iommu_wait_for_completion(struct amd_iommu *iommu)
205 {
206 int ready = 0;
207 unsigned status = 0;
208 unsigned long i = 0;
209
210 while (!ready && (i < EXIT_LOOP_COUNT)) {
211 ++i;
212 /* wait for the bit to become one */
213 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
214 ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
215 }
216
217 /* set bit back to zero */
218 status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
219 writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
220
221 if (unlikely(i == EXIT_LOOP_COUNT))
222 panic("AMD IOMMU: Completion wait loop failed\n");
223 }
224
225 /*
226 * This function queues a completion wait command into the command
227 * buffer of an IOMMU
228 */
229 static int __iommu_completion_wait(struct amd_iommu *iommu)
230 {
231 struct iommu_cmd cmd;
232
233 memset(&cmd, 0, sizeof(cmd));
234 cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
235 CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
236
237 return __iommu_queue_command(iommu, &cmd);
238 }
239
240 /*
241 * This function is called whenever we need to ensure that the IOMMU has
242 * completed execution of all commands we sent. It sends a
243 * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
244 * us about that by writing a value to a physical address we pass with
245 * the command.
246 */
247 static int iommu_completion_wait(struct amd_iommu *iommu)
248 {
249 int ret = 0;
250 unsigned long flags;
251
252 spin_lock_irqsave(&iommu->lock, flags);
253
254 if (!iommu->need_sync)
255 goto out;
256
257 ret = __iommu_completion_wait(iommu);
258
259 iommu->need_sync = 0;
260
261 if (ret)
262 goto out;
263
264 __iommu_wait_for_completion(iommu);
265
266 out:
267 spin_unlock_irqrestore(&iommu->lock, flags);
268
269 return 0;
270 }
271
272 /*
273 * Command send function for invalidating a device table entry
274 */
275 static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
276 {
277 struct iommu_cmd cmd;
278 int ret;
279
280 BUG_ON(iommu == NULL);
281
282 memset(&cmd, 0, sizeof(cmd));
283 CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
284 cmd.data[0] = devid;
285
286 ret = iommu_queue_command(iommu, &cmd);
287
288 return ret;
289 }
290
291 static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
292 u16 domid, int pde, int s)
293 {
294 memset(cmd, 0, sizeof(*cmd));
295 address &= PAGE_MASK;
296 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
297 cmd->data[1] |= domid;
298 cmd->data[2] = lower_32_bits(address);
299 cmd->data[3] = upper_32_bits(address);
300 if (s) /* size bit - we flush more than one 4kb page */
301 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
302 if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
303 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
304 }
305
306 /*
307 * Generic command send function for invalidaing TLB entries
308 */
309 static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
310 u64 address, u16 domid, int pde, int s)
311 {
312 struct iommu_cmd cmd;
313 int ret;
314
315 __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
316
317 ret = iommu_queue_command(iommu, &cmd);
318
319 return ret;
320 }
321
322 /*
323 * TLB invalidation function which is called from the mapping functions.
324 * It invalidates a single PTE if the range to flush is within a single
325 * page. Otherwise it flushes the whole TLB of the IOMMU.
326 */
327 static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
328 u64 address, size_t size)
329 {
330 int s = 0;
331 unsigned pages = iommu_num_pages(address, size, PAGE_SIZE);
332
333 address &= PAGE_MASK;
334
335 if (pages > 1) {
336 /*
337 * If we have to flush more than one page, flush all
338 * TLB entries for this domain
339 */
340 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
341 s = 1;
342 }
343
344 iommu_queue_inv_iommu_pages(iommu, address, domid, 0, s);
345
346 return 0;
347 }
348
349 /* Flush the whole IO/TLB for a given protection domain */
350 static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
351 {
352 u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
353
354 iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1);
355 }
356
357 #ifdef CONFIG_IOMMU_API
358 /*
359 * This function is used to flush the IO/TLB for a given protection domain
360 * on every IOMMU in the system
361 */
362 static void iommu_flush_domain(u16 domid)
363 {
364 unsigned long flags;
365 struct amd_iommu *iommu;
366 struct iommu_cmd cmd;
367
368 __iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
369 domid, 1, 1);
370
371 list_for_each_entry(iommu, &amd_iommu_list, list) {
372 spin_lock_irqsave(&iommu->lock, flags);
373 __iommu_queue_command(iommu, &cmd);
374 __iommu_completion_wait(iommu);
375 __iommu_wait_for_completion(iommu);
376 spin_unlock_irqrestore(&iommu->lock, flags);
377 }
378 }
379 #endif
380
381 /****************************************************************************
382 *
383 * The functions below are used the create the page table mappings for
384 * unity mapped regions.
385 *
386 ****************************************************************************/
387
388 /*
389 * Generic mapping functions. It maps a physical address into a DMA
390 * address space. It allocates the page table pages if necessary.
391 * In the future it can be extended to a generic mapping function
392 * supporting all features of AMD IOMMU page tables like level skipping
393 * and full 64 bit address spaces.
394 */
395 static int iommu_map_page(struct protection_domain *dom,
396 unsigned long bus_addr,
397 unsigned long phys_addr,
398 int prot)
399 {
400 u64 __pte, *pte, *page;
401
402 bus_addr = PAGE_ALIGN(bus_addr);
403 phys_addr = PAGE_ALIGN(phys_addr);
404
405 /* only support 512GB address spaces for now */
406 if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))
407 return -EINVAL;
408
409 pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];
410
411 if (!IOMMU_PTE_PRESENT(*pte)) {
412 page = (u64 *)get_zeroed_page(GFP_KERNEL);
413 if (!page)
414 return -ENOMEM;
415 *pte = IOMMU_L2_PDE(virt_to_phys(page));
416 }
417
418 pte = IOMMU_PTE_PAGE(*pte);
419 pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
420
421 if (!IOMMU_PTE_PRESENT(*pte)) {
422 page = (u64 *)get_zeroed_page(GFP_KERNEL);
423 if (!page)
424 return -ENOMEM;
425 *pte = IOMMU_L1_PDE(virt_to_phys(page));
426 }
427
428 pte = IOMMU_PTE_PAGE(*pte);
429 pte = &pte[IOMMU_PTE_L0_INDEX(bus_addr)];
430
431 if (IOMMU_PTE_PRESENT(*pte))
432 return -EBUSY;
433
434 __pte = phys_addr | IOMMU_PTE_P;
435 if (prot & IOMMU_PROT_IR)
436 __pte |= IOMMU_PTE_IR;
437 if (prot & IOMMU_PROT_IW)
438 __pte |= IOMMU_PTE_IW;
439
440 *pte = __pte;
441
442 return 0;
443 }
444
445 /*
446 * This function checks if a specific unity mapping entry is needed for
447 * this specific IOMMU.
448 */
449 static int iommu_for_unity_map(struct amd_iommu *iommu,
450 struct unity_map_entry *entry)
451 {
452 u16 bdf, i;
453
454 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
455 bdf = amd_iommu_alias_table[i];
456 if (amd_iommu_rlookup_table[bdf] == iommu)
457 return 1;
458 }
459
460 return 0;
461 }
462
463 /*
464 * Init the unity mappings for a specific IOMMU in the system
465 *
466 * Basically iterates over all unity mapping entries and applies them to
467 * the default domain DMA of that IOMMU if necessary.
468 */
469 static int iommu_init_unity_mappings(struct amd_iommu *iommu)
470 {
471 struct unity_map_entry *entry;
472 int ret;
473
474 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
475 if (!iommu_for_unity_map(iommu, entry))
476 continue;
477 ret = dma_ops_unity_map(iommu->default_dom, entry);
478 if (ret)
479 return ret;
480 }
481
482 return 0;
483 }
484
485 /*
486 * This function actually applies the mapping to the page table of the
487 * dma_ops domain.
488 */
489 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
490 struct unity_map_entry *e)
491 {
492 u64 addr;
493 int ret;
494
495 for (addr = e->address_start; addr < e->address_end;
496 addr += PAGE_SIZE) {
497 ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot);
498 if (ret)
499 return ret;
500 /*
501 * if unity mapping is in aperture range mark the page
502 * as allocated in the aperture
503 */
504 if (addr < dma_dom->aperture_size)
505 __set_bit(addr >> PAGE_SHIFT, dma_dom->bitmap);
506 }
507
508 return 0;
509 }
510
511 /*
512 * Inits the unity mappings required for a specific device
513 */
514 static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
515 u16 devid)
516 {
517 struct unity_map_entry *e;
518 int ret;
519
520 list_for_each_entry(e, &amd_iommu_unity_map, list) {
521 if (!(devid >= e->devid_start && devid <= e->devid_end))
522 continue;
523 ret = dma_ops_unity_map(dma_dom, e);
524 if (ret)
525 return ret;
526 }
527
528 return 0;
529 }
530
531 /****************************************************************************
532 *
533 * The next functions belong to the address allocator for the dma_ops
534 * interface functions. They work like the allocators in the other IOMMU
535 * drivers. Its basically a bitmap which marks the allocated pages in
536 * the aperture. Maybe it could be enhanced in the future to a more
537 * efficient allocator.
538 *
539 ****************************************************************************/
540
541 /*
542 * The address allocator core function.
543 *
544 * called with domain->lock held
545 */
546 static unsigned long dma_ops_alloc_addresses(struct device *dev,
547 struct dma_ops_domain *dom,
548 unsigned int pages,
549 unsigned long align_mask,
550 u64 dma_mask)
551 {
552 unsigned long limit;
553 unsigned long address;
554 unsigned long boundary_size;
555
556 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
557 PAGE_SIZE) >> PAGE_SHIFT;
558 limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0,
559 dma_mask >> PAGE_SHIFT);
560
561 if (dom->next_bit >= limit) {
562 dom->next_bit = 0;
563 dom->need_flush = true;
564 }
565
566 address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,
567 0 , boundary_size, align_mask);
568 if (address == -1) {
569 address = iommu_area_alloc(dom->bitmap, limit, 0, pages,
570 0, boundary_size, align_mask);
571 dom->need_flush = true;
572 }
573
574 if (likely(address != -1)) {
575 dom->next_bit = address + pages;
576 address <<= PAGE_SHIFT;
577 } else
578 address = bad_dma_address;
579
580 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
581
582 return address;
583 }
584
585 /*
586 * The address free function.
587 *
588 * called with domain->lock held
589 */
590 static void dma_ops_free_addresses(struct dma_ops_domain *dom,
591 unsigned long address,
592 unsigned int pages)
593 {
594 address >>= PAGE_SHIFT;
595 iommu_area_free(dom->bitmap, address, pages);
596
597 if (address >= dom->next_bit)
598 dom->need_flush = true;
599 }
600
601 /****************************************************************************
602 *
603 * The next functions belong to the domain allocation. A domain is
604 * allocated for every IOMMU as the default domain. If device isolation
605 * is enabled, every device get its own domain. The most important thing
606 * about domains is the page table mapping the DMA address space they
607 * contain.
608 *
609 ****************************************************************************/
610
611 static u16 domain_id_alloc(void)
612 {
613 unsigned long flags;
614 int id;
615
616 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
617 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
618 BUG_ON(id == 0);
619 if (id > 0 && id < MAX_DOMAIN_ID)
620 __set_bit(id, amd_iommu_pd_alloc_bitmap);
621 else
622 id = 0;
623 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
624
625 return id;
626 }
627
628 #ifdef CONFIG_IOMMU_API
629 static void domain_id_free(int id)
630 {
631 unsigned long flags;
632
633 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
634 if (id > 0 && id < MAX_DOMAIN_ID)
635 __clear_bit(id, amd_iommu_pd_alloc_bitmap);
636 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
637 }
638 #endif
639
640 /*
641 * Used to reserve address ranges in the aperture (e.g. for exclusion
642 * ranges.
643 */
644 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
645 unsigned long start_page,
646 unsigned int pages)
647 {
648 unsigned int last_page = dom->aperture_size >> PAGE_SHIFT;
649
650 if (start_page + pages > last_page)
651 pages = last_page - start_page;
652
653 iommu_area_reserve(dom->bitmap, start_page, pages);
654 }
655
656 static void free_pagetable(struct protection_domain *domain)
657 {
658 int i, j;
659 u64 *p1, *p2, *p3;
660
661 p1 = domain->pt_root;
662
663 if (!p1)
664 return;
665
666 for (i = 0; i < 512; ++i) {
667 if (!IOMMU_PTE_PRESENT(p1[i]))
668 continue;
669
670 p2 = IOMMU_PTE_PAGE(p1[i]);
671 for (j = 0; j < 512; ++j) {
672 if (!IOMMU_PTE_PRESENT(p2[j]))
673 continue;
674 p3 = IOMMU_PTE_PAGE(p2[j]);
675 free_page((unsigned long)p3);
676 }
677
678 free_page((unsigned long)p2);
679 }
680
681 free_page((unsigned long)p1);
682
683 domain->pt_root = NULL;
684 }
685
686 /*
687 * Free a domain, only used if something went wrong in the
688 * allocation path and we need to free an already allocated page table
689 */
690 static void dma_ops_domain_free(struct dma_ops_domain *dom)
691 {
692 if (!dom)
693 return;
694
695 free_pagetable(&dom->domain);
696
697 kfree(dom->pte_pages);
698
699 kfree(dom->bitmap);
700
701 kfree(dom);
702 }
703
704 /*
705 * Allocates a new protection domain usable for the dma_ops functions.
706 * It also intializes the page table and the address allocator data
707 * structures required for the dma_ops interface
708 */
709 static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
710 unsigned order)
711 {
712 struct dma_ops_domain *dma_dom;
713 unsigned i, num_pte_pages;
714 u64 *l2_pde;
715 u64 address;
716
717 /*
718 * Currently the DMA aperture must be between 32 MB and 1GB in size
719 */
720 if ((order < 25) || (order > 30))
721 return NULL;
722
723 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
724 if (!dma_dom)
725 return NULL;
726
727 spin_lock_init(&dma_dom->domain.lock);
728
729 dma_dom->domain.id = domain_id_alloc();
730 if (dma_dom->domain.id == 0)
731 goto free_dma_dom;
732 dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
733 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
734 dma_dom->domain.flags = PD_DMA_OPS_MASK;
735 dma_dom->domain.priv = dma_dom;
736 if (!dma_dom->domain.pt_root)
737 goto free_dma_dom;
738 dma_dom->aperture_size = (1ULL << order);
739 dma_dom->bitmap = kzalloc(dma_dom->aperture_size / (PAGE_SIZE * 8),
740 GFP_KERNEL);
741 if (!dma_dom->bitmap)
742 goto free_dma_dom;
743 /*
744 * mark the first page as allocated so we never return 0 as
745 * a valid dma-address. So we can use 0 as error value
746 */
747 dma_dom->bitmap[0] = 1;
748 dma_dom->next_bit = 0;
749
750 dma_dom->need_flush = false;
751 dma_dom->target_dev = 0xffff;
752
753 /* Intialize the exclusion range if necessary */
754 if (iommu->exclusion_start &&
755 iommu->exclusion_start < dma_dom->aperture_size) {
756 unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
757 int pages = iommu_num_pages(iommu->exclusion_start,
758 iommu->exclusion_length,
759 PAGE_SIZE);
760 dma_ops_reserve_addresses(dma_dom, startpage, pages);
761 }
762
763 /*
764 * At the last step, build the page tables so we don't need to
765 * allocate page table pages in the dma_ops mapping/unmapping
766 * path.
767 */
768 num_pte_pages = dma_dom->aperture_size / (PAGE_SIZE * 512);
769 dma_dom->pte_pages = kzalloc(num_pte_pages * sizeof(void *),
770 GFP_KERNEL);
771 if (!dma_dom->pte_pages)
772 goto free_dma_dom;
773
774 l2_pde = (u64 *)get_zeroed_page(GFP_KERNEL);
775 if (l2_pde == NULL)
776 goto free_dma_dom;
777
778 dma_dom->domain.pt_root[0] = IOMMU_L2_PDE(virt_to_phys(l2_pde));
779
780 for (i = 0; i < num_pte_pages; ++i) {
781 dma_dom->pte_pages[i] = (u64 *)get_zeroed_page(GFP_KERNEL);
782 if (!dma_dom->pte_pages[i])
783 goto free_dma_dom;
784 address = virt_to_phys(dma_dom->pte_pages[i]);
785 l2_pde[i] = IOMMU_L1_PDE(address);
786 }
787
788 return dma_dom;
789
790 free_dma_dom:
791 dma_ops_domain_free(dma_dom);
792
793 return NULL;
794 }
795
796 /*
797 * little helper function to check whether a given protection domain is a
798 * dma_ops domain
799 */
800 static bool dma_ops_domain(struct protection_domain *domain)
801 {
802 return domain->flags & PD_DMA_OPS_MASK;
803 }
804
805 /*
806 * Find out the protection domain structure for a given PCI device. This
807 * will give us the pointer to the page table root for example.
808 */
809 static struct protection_domain *domain_for_device(u16 devid)
810 {
811 struct protection_domain *dom;
812 unsigned long flags;
813
814 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
815 dom = amd_iommu_pd_table[devid];
816 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
817
818 return dom;
819 }
820
821 /*
822 * If a device is not yet associated with a domain, this function does
823 * assigns it visible for the hardware
824 */
825 static void attach_device(struct amd_iommu *iommu,
826 struct protection_domain *domain,
827 u16 devid)
828 {
829 unsigned long flags;
830 u64 pte_root = virt_to_phys(domain->pt_root);
831
832 domain->dev_cnt += 1;
833
834 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
835 << DEV_ENTRY_MODE_SHIFT;
836 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
837
838 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
839 amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
840 amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
841 amd_iommu_dev_table[devid].data[2] = domain->id;
842
843 amd_iommu_pd_table[devid] = domain;
844 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
845
846 iommu_queue_inv_dev_entry(iommu, devid);
847 }
848
849 /*
850 * Removes a device from a protection domain (unlocked)
851 */
852 static void __detach_device(struct protection_domain *domain, u16 devid)
853 {
854
855 /* lock domain */
856 spin_lock(&domain->lock);
857
858 /* remove domain from the lookup table */
859 amd_iommu_pd_table[devid] = NULL;
860
861 /* remove entry from the device table seen by the hardware */
862 amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
863 amd_iommu_dev_table[devid].data[1] = 0;
864 amd_iommu_dev_table[devid].data[2] = 0;
865
866 /* decrease reference counter */
867 domain->dev_cnt -= 1;
868
869 /* ready */
870 spin_unlock(&domain->lock);
871 }
872
873 /*
874 * Removes a device from a protection domain (with devtable_lock held)
875 */
876 static void detach_device(struct protection_domain *domain, u16 devid)
877 {
878 unsigned long flags;
879
880 /* lock device table */
881 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
882 __detach_device(domain, devid);
883 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
884 }
885
886 static int device_change_notifier(struct notifier_block *nb,
887 unsigned long action, void *data)
888 {
889 struct device *dev = data;
890 struct pci_dev *pdev = to_pci_dev(dev);
891 u16 devid = calc_devid(pdev->bus->number, pdev->devfn);
892 struct protection_domain *domain;
893 struct dma_ops_domain *dma_domain;
894 struct amd_iommu *iommu;
895
896 if (devid > amd_iommu_last_bdf)
897 goto out;
898
899 devid = amd_iommu_alias_table[devid];
900
901 iommu = amd_iommu_rlookup_table[devid];
902 if (iommu == NULL)
903 goto out;
904
905 domain = domain_for_device(devid);
906
907 if (domain && !dma_ops_domain(domain))
908 WARN_ONCE(1, "AMD IOMMU WARNING: device %s already bound "
909 "to a non-dma-ops domain\n", dev_name(dev));
910
911 switch (action) {
912 case BUS_NOTIFY_BOUND_DRIVER:
913 if (domain)
914 goto out;
915 dma_domain = find_protection_domain(devid);
916 if (!dma_domain)
917 dma_domain = iommu->default_dom;
918 attach_device(iommu, &dma_domain->domain, devid);
919 printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
920 "device %s\n", dma_domain->domain.id, dev_name(dev));
921 break;
922 case BUS_NOTIFY_UNBIND_DRIVER:
923 if (!domain)
924 goto out;
925 detach_device(domain, devid);
926 break;
927 default:
928 goto out;
929 }
930
931 iommu_queue_inv_dev_entry(iommu, devid);
932 iommu_completion_wait(iommu);
933
934 out:
935 return 0;
936 }
937
938 struct notifier_block device_nb = {
939 .notifier_call = device_change_notifier,
940 };
941
942 /*****************************************************************************
943 *
944 * The next functions belong to the dma_ops mapping/unmapping code.
945 *
946 *****************************************************************************/
947
948 /*
949 * This function checks if the driver got a valid device from the caller to
950 * avoid dereferencing invalid pointers.
951 */
952 static bool check_device(struct device *dev)
953 {
954 if (!dev || !dev->dma_mask)
955 return false;
956
957 return true;
958 }
959
960 /*
961 * In this function the list of preallocated protection domains is traversed to
962 * find the domain for a specific device
963 */
964 static struct dma_ops_domain *find_protection_domain(u16 devid)
965 {
966 struct dma_ops_domain *entry, *ret = NULL;
967 unsigned long flags;
968
969 if (list_empty(&iommu_pd_list))
970 return NULL;
971
972 spin_lock_irqsave(&iommu_pd_list_lock, flags);
973
974 list_for_each_entry(entry, &iommu_pd_list, list) {
975 if (entry->target_dev == devid) {
976 ret = entry;
977 break;
978 }
979 }
980
981 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
982
983 return ret;
984 }
985
986 /*
987 * In the dma_ops path we only have the struct device. This function
988 * finds the corresponding IOMMU, the protection domain and the
989 * requestor id for a given device.
990 * If the device is not yet associated with a domain this is also done
991 * in this function.
992 */
993 static int get_device_resources(struct device *dev,
994 struct amd_iommu **iommu,
995 struct protection_domain **domain,
996 u16 *bdf)
997 {
998 struct dma_ops_domain *dma_dom;
999 struct pci_dev *pcidev;
1000 u16 _bdf;
1001
1002 *iommu = NULL;
1003 *domain = NULL;
1004 *bdf = 0xffff;
1005
1006 if (dev->bus != &pci_bus_type)
1007 return 0;
1008
1009 pcidev = to_pci_dev(dev);
1010 _bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
1011
1012 /* device not translated by any IOMMU in the system? */
1013 if (_bdf > amd_iommu_last_bdf)
1014 return 0;
1015
1016 *bdf = amd_iommu_alias_table[_bdf];
1017
1018 *iommu = amd_iommu_rlookup_table[*bdf];
1019 if (*iommu == NULL)
1020 return 0;
1021 *domain = domain_for_device(*bdf);
1022 if (*domain == NULL) {
1023 dma_dom = find_protection_domain(*bdf);
1024 if (!dma_dom)
1025 dma_dom = (*iommu)->default_dom;
1026 *domain = &dma_dom->domain;
1027 attach_device(*iommu, *domain, *bdf);
1028 printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
1029 "device ", (*domain)->id);
1030 print_devid(_bdf, 1);
1031 }
1032
1033 if (domain_for_device(_bdf) == NULL)
1034 attach_device(*iommu, *domain, _bdf);
1035
1036 return 1;
1037 }
1038
1039 /*
1040 * This is the generic map function. It maps one 4kb page at paddr to
1041 * the given address in the DMA address space for the domain.
1042 */
1043 static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu,
1044 struct dma_ops_domain *dom,
1045 unsigned long address,
1046 phys_addr_t paddr,
1047 int direction)
1048 {
1049 u64 *pte, __pte;
1050
1051 WARN_ON(address > dom->aperture_size);
1052
1053 paddr &= PAGE_MASK;
1054
1055 pte = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
1056 pte += IOMMU_PTE_L0_INDEX(address);
1057
1058 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
1059
1060 if (direction == DMA_TO_DEVICE)
1061 __pte |= IOMMU_PTE_IR;
1062 else if (direction == DMA_FROM_DEVICE)
1063 __pte |= IOMMU_PTE_IW;
1064 else if (direction == DMA_BIDIRECTIONAL)
1065 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
1066
1067 WARN_ON(*pte);
1068
1069 *pte = __pte;
1070
1071 return (dma_addr_t)address;
1072 }
1073
1074 /*
1075 * The generic unmapping function for on page in the DMA address space.
1076 */
1077 static void dma_ops_domain_unmap(struct amd_iommu *iommu,
1078 struct dma_ops_domain *dom,
1079 unsigned long address)
1080 {
1081 u64 *pte;
1082
1083 if (address >= dom->aperture_size)
1084 return;
1085
1086 WARN_ON(address & ~PAGE_MASK || address >= dom->aperture_size);
1087
1088 pte = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
1089 pte += IOMMU_PTE_L0_INDEX(address);
1090
1091 WARN_ON(!*pte);
1092
1093 *pte = 0ULL;
1094 }
1095
1096 /*
1097 * This function contains common code for mapping of a physically
1098 * contiguous memory region into DMA address space. It is used by all
1099 * mapping functions provided with this IOMMU driver.
1100 * Must be called with the domain lock held.
1101 */
1102 static dma_addr_t __map_single(struct device *dev,
1103 struct amd_iommu *iommu,
1104 struct dma_ops_domain *dma_dom,
1105 phys_addr_t paddr,
1106 size_t size,
1107 int dir,
1108 bool align,
1109 u64 dma_mask)
1110 {
1111 dma_addr_t offset = paddr & ~PAGE_MASK;
1112 dma_addr_t address, start;
1113 unsigned int pages;
1114 unsigned long align_mask = 0;
1115 int i;
1116
1117 pages = iommu_num_pages(paddr, size, PAGE_SIZE);
1118 paddr &= PAGE_MASK;
1119
1120 if (align)
1121 align_mask = (1UL << get_order(size)) - 1;
1122
1123 address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
1124 dma_mask);
1125 if (unlikely(address == bad_dma_address))
1126 goto out;
1127
1128 start = address;
1129 for (i = 0; i < pages; ++i) {
1130 dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
1131 paddr += PAGE_SIZE;
1132 start += PAGE_SIZE;
1133 }
1134 address += offset;
1135
1136 if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
1137 iommu_flush_tlb(iommu, dma_dom->domain.id);
1138 dma_dom->need_flush = false;
1139 } else if (unlikely(iommu_has_npcache(iommu)))
1140 iommu_flush_pages(iommu, dma_dom->domain.id, address, size);
1141
1142 out:
1143 return address;
1144 }
1145
1146 /*
1147 * Does the reverse of the __map_single function. Must be called with
1148 * the domain lock held too
1149 */
1150 static void __unmap_single(struct amd_iommu *iommu,
1151 struct dma_ops_domain *dma_dom,
1152 dma_addr_t dma_addr,
1153 size_t size,
1154 int dir)
1155 {
1156 dma_addr_t i, start;
1157 unsigned int pages;
1158
1159 if ((dma_addr == bad_dma_address) ||
1160 (dma_addr + size > dma_dom->aperture_size))
1161 return;
1162
1163 pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
1164 dma_addr &= PAGE_MASK;
1165 start = dma_addr;
1166
1167 for (i = 0; i < pages; ++i) {
1168 dma_ops_domain_unmap(iommu, dma_dom, start);
1169 start += PAGE_SIZE;
1170 }
1171
1172 dma_ops_free_addresses(dma_dom, dma_addr, pages);
1173
1174 if (amd_iommu_unmap_flush || dma_dom->need_flush) {
1175 iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
1176 dma_dom->need_flush = false;
1177 }
1178 }
1179
1180 /*
1181 * The exported map_single function for dma_ops.
1182 */
1183 static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
1184 size_t size, int dir)
1185 {
1186 unsigned long flags;
1187 struct amd_iommu *iommu;
1188 struct protection_domain *domain;
1189 u16 devid;
1190 dma_addr_t addr;
1191 u64 dma_mask;
1192
1193 if (!check_device(dev))
1194 return bad_dma_address;
1195
1196 dma_mask = *dev->dma_mask;
1197
1198 get_device_resources(dev, &iommu, &domain, &devid);
1199
1200 if (iommu == NULL || domain == NULL)
1201 /* device not handled by any AMD IOMMU */
1202 return (dma_addr_t)paddr;
1203
1204 if (!dma_ops_domain(domain))
1205 return bad_dma_address;
1206
1207 spin_lock_irqsave(&domain->lock, flags);
1208 addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
1209 dma_mask);
1210 if (addr == bad_dma_address)
1211 goto out;
1212
1213 iommu_completion_wait(iommu);
1214
1215 out:
1216 spin_unlock_irqrestore(&domain->lock, flags);
1217
1218 return addr;
1219 }
1220
1221 /*
1222 * The exported unmap_single function for dma_ops.
1223 */
1224 static void unmap_single(struct device *dev, dma_addr_t dma_addr,
1225 size_t size, int dir)
1226 {
1227 unsigned long flags;
1228 struct amd_iommu *iommu;
1229 struct protection_domain *domain;
1230 u16 devid;
1231
1232 if (!check_device(dev) ||
1233 !get_device_resources(dev, &iommu, &domain, &devid))
1234 /* device not handled by any AMD IOMMU */
1235 return;
1236
1237 if (!dma_ops_domain(domain))
1238 return;
1239
1240 spin_lock_irqsave(&domain->lock, flags);
1241
1242 __unmap_single(iommu, domain->priv, dma_addr, size, dir);
1243
1244 iommu_completion_wait(iommu);
1245
1246 spin_unlock_irqrestore(&domain->lock, flags);
1247 }
1248
1249 /*
1250 * This is a special map_sg function which is used if we should map a
1251 * device which is not handled by an AMD IOMMU in the system.
1252 */
1253 static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
1254 int nelems, int dir)
1255 {
1256 struct scatterlist *s;
1257 int i;
1258
1259 for_each_sg(sglist, s, nelems, i) {
1260 s->dma_address = (dma_addr_t)sg_phys(s);
1261 s->dma_length = s->length;
1262 }
1263
1264 return nelems;
1265 }
1266
1267 /*
1268 * The exported map_sg function for dma_ops (handles scatter-gather
1269 * lists).
1270 */
1271 static int map_sg(struct device *dev, struct scatterlist *sglist,
1272 int nelems, int dir)
1273 {
1274 unsigned long flags;
1275 struct amd_iommu *iommu;
1276 struct protection_domain *domain;
1277 u16 devid;
1278 int i;
1279 struct scatterlist *s;
1280 phys_addr_t paddr;
1281 int mapped_elems = 0;
1282 u64 dma_mask;
1283
1284 if (!check_device(dev))
1285 return 0;
1286
1287 dma_mask = *dev->dma_mask;
1288
1289 get_device_resources(dev, &iommu, &domain, &devid);
1290
1291 if (!iommu || !domain)
1292 return map_sg_no_iommu(dev, sglist, nelems, dir);
1293
1294 if (!dma_ops_domain(domain))
1295 return 0;
1296
1297 spin_lock_irqsave(&domain->lock, flags);
1298
1299 for_each_sg(sglist, s, nelems, i) {
1300 paddr = sg_phys(s);
1301
1302 s->dma_address = __map_single(dev, iommu, domain->priv,
1303 paddr, s->length, dir, false,
1304 dma_mask);
1305
1306 if (s->dma_address) {
1307 s->dma_length = s->length;
1308 mapped_elems++;
1309 } else
1310 goto unmap;
1311 }
1312
1313 iommu_completion_wait(iommu);
1314
1315 out:
1316 spin_unlock_irqrestore(&domain->lock, flags);
1317
1318 return mapped_elems;
1319 unmap:
1320 for_each_sg(sglist, s, mapped_elems, i) {
1321 if (s->dma_address)
1322 __unmap_single(iommu, domain->priv, s->dma_address,
1323 s->dma_length, dir);
1324 s->dma_address = s->dma_length = 0;
1325 }
1326
1327 mapped_elems = 0;
1328
1329 goto out;
1330 }
1331
1332 /*
1333 * The exported map_sg function for dma_ops (handles scatter-gather
1334 * lists).
1335 */
1336 static void unmap_sg(struct device *dev, struct scatterlist *sglist,
1337 int nelems, int dir)
1338 {
1339 unsigned long flags;
1340 struct amd_iommu *iommu;
1341 struct protection_domain *domain;
1342 struct scatterlist *s;
1343 u16 devid;
1344 int i;
1345
1346 if (!check_device(dev) ||
1347 !get_device_resources(dev, &iommu, &domain, &devid))
1348 return;
1349
1350 if (!dma_ops_domain(domain))
1351 return;
1352
1353 spin_lock_irqsave(&domain->lock, flags);
1354
1355 for_each_sg(sglist, s, nelems, i) {
1356 __unmap_single(iommu, domain->priv, s->dma_address,
1357 s->dma_length, dir);
1358 s->dma_address = s->dma_length = 0;
1359 }
1360
1361 iommu_completion_wait(iommu);
1362
1363 spin_unlock_irqrestore(&domain->lock, flags);
1364 }
1365
1366 /*
1367 * The exported alloc_coherent function for dma_ops.
1368 */
1369 static void *alloc_coherent(struct device *dev, size_t size,
1370 dma_addr_t *dma_addr, gfp_t flag)
1371 {
1372 unsigned long flags;
1373 void *virt_addr;
1374 struct amd_iommu *iommu;
1375 struct protection_domain *domain;
1376 u16 devid;
1377 phys_addr_t paddr;
1378 u64 dma_mask = dev->coherent_dma_mask;
1379
1380 if (!check_device(dev))
1381 return NULL;
1382
1383 if (!get_device_resources(dev, &iommu, &domain, &devid))
1384 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
1385
1386 flag |= __GFP_ZERO;
1387 virt_addr = (void *)__get_free_pages(flag, get_order(size));
1388 if (!virt_addr)
1389 return 0;
1390
1391 paddr = virt_to_phys(virt_addr);
1392
1393 if (!iommu || !domain) {
1394 *dma_addr = (dma_addr_t)paddr;
1395 return virt_addr;
1396 }
1397
1398 if (!dma_ops_domain(domain))
1399 goto out_free;
1400
1401 if (!dma_mask)
1402 dma_mask = *dev->dma_mask;
1403
1404 spin_lock_irqsave(&domain->lock, flags);
1405
1406 *dma_addr = __map_single(dev, iommu, domain->priv, paddr,
1407 size, DMA_BIDIRECTIONAL, true, dma_mask);
1408
1409 if (*dma_addr == bad_dma_address)
1410 goto out_free;
1411
1412 iommu_completion_wait(iommu);
1413
1414 spin_unlock_irqrestore(&domain->lock, flags);
1415
1416 return virt_addr;
1417
1418 out_free:
1419
1420 free_pages((unsigned long)virt_addr, get_order(size));
1421
1422 return NULL;
1423 }
1424
1425 /*
1426 * The exported free_coherent function for dma_ops.
1427 */
1428 static void free_coherent(struct device *dev, size_t size,
1429 void *virt_addr, dma_addr_t dma_addr)
1430 {
1431 unsigned long flags;
1432 struct amd_iommu *iommu;
1433 struct protection_domain *domain;
1434 u16 devid;
1435
1436 if (!check_device(dev))
1437 return;
1438
1439 get_device_resources(dev, &iommu, &domain, &devid);
1440
1441 if (!iommu || !domain)
1442 goto free_mem;
1443
1444 if (!dma_ops_domain(domain))
1445 goto free_mem;
1446
1447 spin_lock_irqsave(&domain->lock, flags);
1448
1449 __unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
1450
1451 iommu_completion_wait(iommu);
1452
1453 spin_unlock_irqrestore(&domain->lock, flags);
1454
1455 free_mem:
1456 free_pages((unsigned long)virt_addr, get_order(size));
1457 }
1458
1459 /*
1460 * This function is called by the DMA layer to find out if we can handle a
1461 * particular device. It is part of the dma_ops.
1462 */
1463 static int amd_iommu_dma_supported(struct device *dev, u64 mask)
1464 {
1465 u16 bdf;
1466 struct pci_dev *pcidev;
1467
1468 /* No device or no PCI device */
1469 if (!dev || dev->bus != &pci_bus_type)
1470 return 0;
1471
1472 pcidev = to_pci_dev(dev);
1473
1474 bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
1475
1476 /* Out of our scope? */
1477 if (bdf > amd_iommu_last_bdf)
1478 return 0;
1479
1480 return 1;
1481 }
1482
1483 /*
1484 * The function for pre-allocating protection domains.
1485 *
1486 * If the driver core informs the DMA layer if a driver grabs a device
1487 * we don't need to preallocate the protection domains anymore.
1488 * For now we have to.
1489 */
1490 void prealloc_protection_domains(void)
1491 {
1492 struct pci_dev *dev = NULL;
1493 struct dma_ops_domain *dma_dom;
1494 struct amd_iommu *iommu;
1495 int order = amd_iommu_aperture_order;
1496 u16 devid;
1497
1498 while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
1499 devid = (dev->bus->number << 8) | dev->devfn;
1500 if (devid > amd_iommu_last_bdf)
1501 continue;
1502 devid = amd_iommu_alias_table[devid];
1503 if (domain_for_device(devid))
1504 continue;
1505 iommu = amd_iommu_rlookup_table[devid];
1506 if (!iommu)
1507 continue;
1508 dma_dom = dma_ops_domain_alloc(iommu, order);
1509 if (!dma_dom)
1510 continue;
1511 init_unity_mappings_for_device(dma_dom, devid);
1512 dma_dom->target_dev = devid;
1513
1514 list_add_tail(&dma_dom->list, &iommu_pd_list);
1515 }
1516 }
1517
1518 static struct dma_mapping_ops amd_iommu_dma_ops = {
1519 .alloc_coherent = alloc_coherent,
1520 .free_coherent = free_coherent,
1521 .map_single = map_single,
1522 .unmap_single = unmap_single,
1523 .map_sg = map_sg,
1524 .unmap_sg = unmap_sg,
1525 .dma_supported = amd_iommu_dma_supported,
1526 };
1527
1528 /*
1529 * The function which clues the AMD IOMMU driver into dma_ops.
1530 */
1531 int __init amd_iommu_init_dma_ops(void)
1532 {
1533 struct amd_iommu *iommu;
1534 int order = amd_iommu_aperture_order;
1535 int ret;
1536
1537 /*
1538 * first allocate a default protection domain for every IOMMU we
1539 * found in the system. Devices not assigned to any other
1540 * protection domain will be assigned to the default one.
1541 */
1542 list_for_each_entry(iommu, &amd_iommu_list, list) {
1543 iommu->default_dom = dma_ops_domain_alloc(iommu, order);
1544 if (iommu->default_dom == NULL)
1545 return -ENOMEM;
1546 ret = iommu_init_unity_mappings(iommu);
1547 if (ret)
1548 goto free_domains;
1549 }
1550
1551 /*
1552 * If device isolation is enabled, pre-allocate the protection
1553 * domains for each device.
1554 */
1555 if (amd_iommu_isolate)
1556 prealloc_protection_domains();
1557
1558 iommu_detected = 1;
1559 force_iommu = 1;
1560 bad_dma_address = 0;
1561 #ifdef CONFIG_GART_IOMMU
1562 gart_iommu_aperture_disabled = 1;
1563 gart_iommu_aperture = 0;
1564 #endif
1565
1566 /* Make the driver finally visible to the drivers */
1567 dma_ops = &amd_iommu_dma_ops;
1568
1569 bus_register_notifier(&pci_bus_type, &device_nb);
1570
1571 return 0;
1572
1573 free_domains:
1574
1575 list_for_each_entry(iommu, &amd_iommu_list, list) {
1576 if (iommu->default_dom)
1577 dma_ops_domain_free(iommu->default_dom);
1578 }
1579
1580 return ret;
1581 }
1582
1583 /*****************************************************************************
1584 *
1585 * The following functions belong to the exported interface of AMD IOMMU
1586 *
1587 * This interface allows access to lower level functions of the IOMMU
1588 * like protection domain handling and assignement of devices to domains
1589 * which is not possible with the dma_ops interface.
1590 *
1591 *****************************************************************************/
1592
1593 #ifdef CONFIG_IOMMU_API
1594
1595 static void cleanup_domain(struct protection_domain *domain)
1596 {
1597 unsigned long flags;
1598 u16 devid;
1599
1600 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1601
1602 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
1603 if (amd_iommu_pd_table[devid] == domain)
1604 __detach_device(domain, devid);
1605
1606 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1607 }
1608
1609 #endif
Linux kernel - Deliverables
This page took 0.488435 seconds and 5 git commands to generate.