1 /* pci_sun4v.c: SUN4V specific PCI controller support.
3 * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net)
6 #include <linux/kernel.h>
7 #include <linux/types.h>
9 #include <linux/init.h>
10 #include <linux/slab.h>
11 #include <linux/interrupt.h>
12 #include <linux/percpu.h>
13 #include <linux/irq.h>
14 #include <linux/msi.h>
15 #include <linux/log2.h>
17 #include <asm/iommu.h>
20 #include <asm/pstate.h>
21 #include <asm/oplib.h>
22 #include <asm/hypervisor.h>
26 #include "iommu_common.h"
28 #include "pci_sun4v.h"
30 static unsigned long vpci_major
= 1;
31 static unsigned long vpci_minor
= 1;
33 #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64))
36 struct device
*dev
; /* Device mapping is for. */
37 unsigned long prot
; /* IOMMU page protections */
38 unsigned long entry
; /* Index into IOTSB. */
39 u64
*pglist
; /* List of physical pages */
40 unsigned long npages
; /* Number of pages in list. */
43 static DEFINE_PER_CPU(struct iommu_batch
, iommu_batch
);
45 /* Interrupts must be disabled. */
46 static inline void iommu_batch_start(struct device
*dev
, unsigned long prot
, unsigned long entry
)
48 struct iommu_batch
*p
= &__get_cpu_var(iommu_batch
);
56 /* Interrupts must be disabled. */
57 static long iommu_batch_flush(struct iommu_batch
*p
)
59 struct pci_pbm_info
*pbm
= p
->dev
->archdata
.host_controller
;
60 unsigned long devhandle
= pbm
->devhandle
;
61 unsigned long prot
= p
->prot
;
62 unsigned long entry
= p
->entry
;
63 u64
*pglist
= p
->pglist
;
64 unsigned long npages
= p
->npages
;
69 num
= pci_sun4v_iommu_map(devhandle
, HV_PCI_TSBID(0, entry
),
70 npages
, prot
, __pa(pglist
));
71 if (unlikely(num
< 0)) {
72 if (printk_ratelimit())
73 printk("iommu_batch_flush: IOMMU map of "
74 "[%08lx:%08lx:%lx:%lx:%lx] failed with "
76 devhandle
, HV_PCI_TSBID(0, entry
),
77 npages
, prot
, __pa(pglist
), num
);
92 static inline void iommu_batch_new_entry(unsigned long entry
)
94 struct iommu_batch
*p
= &__get_cpu_var(iommu_batch
);
96 if (p
->entry
+ p
->npages
== entry
)
103 /* Interrupts must be disabled. */
104 static inline long iommu_batch_add(u64 phys_page
)
106 struct iommu_batch
*p
= &__get_cpu_var(iommu_batch
);
108 BUG_ON(p
->npages
>= PGLIST_NENTS
);
110 p
->pglist
[p
->npages
++] = phys_page
;
111 if (p
->npages
== PGLIST_NENTS
)
112 return iommu_batch_flush(p
);
117 /* Interrupts must be disabled. */
118 static inline long iommu_batch_end(void)
120 struct iommu_batch
*p
= &__get_cpu_var(iommu_batch
);
122 BUG_ON(p
->npages
>= PGLIST_NENTS
);
124 return iommu_batch_flush(p
);
127 static void *dma_4v_alloc_coherent(struct device
*dev
, size_t size
,
128 dma_addr_t
*dma_addrp
, gfp_t gfp
)
131 unsigned long flags
, order
, first_page
, npages
, n
;
135 size
= IO_PAGE_ALIGN(size
);
136 order
= get_order(size
);
137 if (unlikely(order
>= MAX_ORDER
))
140 npages
= size
>> IO_PAGE_SHIFT
;
142 first_page
= __get_free_pages(gfp
, order
);
143 if (unlikely(first_page
== 0UL))
146 memset((char *)first_page
, 0, PAGE_SIZE
<< order
);
148 iommu
= dev
->archdata
.iommu
;
150 spin_lock_irqsave(&iommu
->lock
, flags
);
151 entry
= iommu_range_alloc(dev
, iommu
, npages
, NULL
);
152 spin_unlock_irqrestore(&iommu
->lock
, flags
);
154 if (unlikely(entry
== DMA_ERROR_CODE
))
155 goto range_alloc_fail
;
157 *dma_addrp
= (iommu
->page_table_map_base
+
158 (entry
<< IO_PAGE_SHIFT
));
159 ret
= (void *) first_page
;
160 first_page
= __pa(first_page
);
162 local_irq_save(flags
);
164 iommu_batch_start(dev
,
165 (HV_PCI_MAP_ATTR_READ
|
166 HV_PCI_MAP_ATTR_WRITE
),
169 for (n
= 0; n
< npages
; n
++) {
170 long err
= iommu_batch_add(first_page
+ (n
* PAGE_SIZE
));
171 if (unlikely(err
< 0L))
175 if (unlikely(iommu_batch_end() < 0L))
178 local_irq_restore(flags
);
183 /* Interrupts are disabled. */
184 spin_lock(&iommu
->lock
);
185 iommu_range_free(iommu
, *dma_addrp
, npages
);
186 spin_unlock_irqrestore(&iommu
->lock
, flags
);
189 free_pages(first_page
, order
);
193 static void dma_4v_free_coherent(struct device
*dev
, size_t size
, void *cpu
,
196 struct pci_pbm_info
*pbm
;
198 unsigned long flags
, order
, npages
, entry
;
201 npages
= IO_PAGE_ALIGN(size
) >> IO_PAGE_SHIFT
;
202 iommu
= dev
->archdata
.iommu
;
203 pbm
= dev
->archdata
.host_controller
;
204 devhandle
= pbm
->devhandle
;
205 entry
= ((dvma
- iommu
->page_table_map_base
) >> IO_PAGE_SHIFT
);
207 spin_lock_irqsave(&iommu
->lock
, flags
);
209 iommu_range_free(iommu
, dvma
, npages
);
214 num
= pci_sun4v_iommu_demap(devhandle
, HV_PCI_TSBID(0, entry
),
218 } while (npages
!= 0);
220 spin_unlock_irqrestore(&iommu
->lock
, flags
);
222 order
= get_order(size
);
224 free_pages((unsigned long)cpu
, order
);
227 static dma_addr_t
dma_4v_map_single(struct device
*dev
, void *ptr
, size_t sz
,
228 enum dma_data_direction direction
)
231 unsigned long flags
, npages
, oaddr
;
232 unsigned long i
, base_paddr
;
237 iommu
= dev
->archdata
.iommu
;
239 if (unlikely(direction
== DMA_NONE
))
242 oaddr
= (unsigned long)ptr
;
243 npages
= IO_PAGE_ALIGN(oaddr
+ sz
) - (oaddr
& IO_PAGE_MASK
);
244 npages
>>= IO_PAGE_SHIFT
;
246 spin_lock_irqsave(&iommu
->lock
, flags
);
247 entry
= iommu_range_alloc(dev
, iommu
, npages
, NULL
);
248 spin_unlock_irqrestore(&iommu
->lock
, flags
);
250 if (unlikely(entry
== DMA_ERROR_CODE
))
253 bus_addr
= (iommu
->page_table_map_base
+
254 (entry
<< IO_PAGE_SHIFT
));
255 ret
= bus_addr
| (oaddr
& ~IO_PAGE_MASK
);
256 base_paddr
= __pa(oaddr
& IO_PAGE_MASK
);
257 prot
= HV_PCI_MAP_ATTR_READ
;
258 if (direction
!= DMA_TO_DEVICE
)
259 prot
|= HV_PCI_MAP_ATTR_WRITE
;
261 local_irq_save(flags
);
263 iommu_batch_start(dev
, prot
, entry
);
265 for (i
= 0; i
< npages
; i
++, base_paddr
+= IO_PAGE_SIZE
) {
266 long err
= iommu_batch_add(base_paddr
);
267 if (unlikely(err
< 0L))
270 if (unlikely(iommu_batch_end() < 0L))
273 local_irq_restore(flags
);
278 if (printk_ratelimit())
280 return DMA_ERROR_CODE
;
283 /* Interrupts are disabled. */
284 spin_lock(&iommu
->lock
);
285 iommu_range_free(iommu
, bus_addr
, npages
);
286 spin_unlock_irqrestore(&iommu
->lock
, flags
);
288 return DMA_ERROR_CODE
;
291 static void dma_4v_unmap_single(struct device
*dev
, dma_addr_t bus_addr
,
292 size_t sz
, enum dma_data_direction direction
)
294 struct pci_pbm_info
*pbm
;
296 unsigned long flags
, npages
;
300 if (unlikely(direction
== DMA_NONE
)) {
301 if (printk_ratelimit())
306 iommu
= dev
->archdata
.iommu
;
307 pbm
= dev
->archdata
.host_controller
;
308 devhandle
= pbm
->devhandle
;
310 npages
= IO_PAGE_ALIGN(bus_addr
+ sz
) - (bus_addr
& IO_PAGE_MASK
);
311 npages
>>= IO_PAGE_SHIFT
;
312 bus_addr
&= IO_PAGE_MASK
;
314 spin_lock_irqsave(&iommu
->lock
, flags
);
316 iommu_range_free(iommu
, bus_addr
, npages
);
318 entry
= (bus_addr
- iommu
->page_table_map_base
) >> IO_PAGE_SHIFT
;
322 num
= pci_sun4v_iommu_demap(devhandle
, HV_PCI_TSBID(0, entry
),
326 } while (npages
!= 0);
328 spin_unlock_irqrestore(&iommu
->lock
, flags
);
331 static int dma_4v_map_sg(struct device
*dev
, struct scatterlist
*sglist
,
332 int nelems
, enum dma_data_direction direction
)
334 struct scatterlist
*s
, *outs
, *segstart
;
335 unsigned long flags
, handle
, prot
;
336 dma_addr_t dma_next
= 0, dma_addr
;
337 unsigned int max_seg_size
;
338 unsigned long seg_boundary_size
;
339 int outcount
, incount
, i
;
341 unsigned long base_shift
;
344 BUG_ON(direction
== DMA_NONE
);
346 iommu
= dev
->archdata
.iommu
;
347 if (nelems
== 0 || !iommu
)
350 prot
= HV_PCI_MAP_ATTR_READ
;
351 if (direction
!= DMA_TO_DEVICE
)
352 prot
|= HV_PCI_MAP_ATTR_WRITE
;
354 outs
= s
= segstart
= &sglist
[0];
359 /* Init first segment length for backout at failure */
360 outs
->dma_length
= 0;
362 spin_lock_irqsave(&iommu
->lock
, flags
);
364 iommu_batch_start(dev
, prot
, ~0UL);
366 max_seg_size
= dma_get_max_seg_size(dev
);
367 seg_boundary_size
= ALIGN(dma_get_seg_boundary(dev
) + 1,
368 IO_PAGE_SIZE
) >> IO_PAGE_SHIFT
;
369 base_shift
= iommu
->page_table_map_base
>> IO_PAGE_SHIFT
;
370 for_each_sg(sglist
, s
, nelems
, i
) {
371 unsigned long paddr
, npages
, entry
, out_entry
= 0, slen
;
379 /* Allocate iommu entries for that segment */
380 paddr
= (unsigned long) SG_ENT_PHYS_ADDRESS(s
);
381 npages
= iommu_num_pages(paddr
, slen
);
382 entry
= iommu_range_alloc(dev
, iommu
, npages
, &handle
);
385 if (unlikely(entry
== DMA_ERROR_CODE
)) {
386 if (printk_ratelimit())
387 printk(KERN_INFO
"iommu_alloc failed, iommu %p paddr %lx"
388 " npages %lx\n", iommu
, paddr
, npages
);
389 goto iommu_map_failed
;
392 iommu_batch_new_entry(entry
);
394 /* Convert entry to a dma_addr_t */
395 dma_addr
= iommu
->page_table_map_base
+
396 (entry
<< IO_PAGE_SHIFT
);
397 dma_addr
|= (s
->offset
& ~IO_PAGE_MASK
);
399 /* Insert into HW table */
400 paddr
&= IO_PAGE_MASK
;
402 err
= iommu_batch_add(paddr
);
403 if (unlikely(err
< 0L))
404 goto iommu_map_failed
;
405 paddr
+= IO_PAGE_SIZE
;
408 /* If we are in an open segment, try merging */
410 /* We cannot merge if:
411 * - allocated dma_addr isn't contiguous to previous allocation
413 if ((dma_addr
!= dma_next
) ||
414 (outs
->dma_length
+ s
->length
> max_seg_size
) ||
415 (is_span_boundary(out_entry
, base_shift
,
416 seg_boundary_size
, outs
, s
))) {
417 /* Can't merge: create a new segment */
420 outs
= sg_next(outs
);
422 outs
->dma_length
+= s
->length
;
427 /* This is a new segment, fill entries */
428 outs
->dma_address
= dma_addr
;
429 outs
->dma_length
= slen
;
433 /* Calculate next page pointer for contiguous check */
434 dma_next
= dma_addr
+ slen
;
437 err
= iommu_batch_end();
439 if (unlikely(err
< 0L))
440 goto iommu_map_failed
;
442 spin_unlock_irqrestore(&iommu
->lock
, flags
);
444 if (outcount
< incount
) {
445 outs
= sg_next(outs
);
446 outs
->dma_address
= DMA_ERROR_CODE
;
447 outs
->dma_length
= 0;
453 for_each_sg(sglist
, s
, nelems
, i
) {
454 if (s
->dma_length
!= 0) {
455 unsigned long vaddr
, npages
;
457 vaddr
= s
->dma_address
& IO_PAGE_MASK
;
458 npages
= iommu_num_pages(s
->dma_address
, s
->dma_length
);
459 iommu_range_free(iommu
, vaddr
, npages
);
461 s
->dma_address
= DMA_ERROR_CODE
;
467 spin_unlock_irqrestore(&iommu
->lock
, flags
);
472 static void dma_4v_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
473 int nelems
, enum dma_data_direction direction
)
475 struct pci_pbm_info
*pbm
;
476 struct scatterlist
*sg
;
481 BUG_ON(direction
== DMA_NONE
);
483 iommu
= dev
->archdata
.iommu
;
484 pbm
= dev
->archdata
.host_controller
;
485 devhandle
= pbm
->devhandle
;
487 spin_lock_irqsave(&iommu
->lock
, flags
);
491 dma_addr_t dma_handle
= sg
->dma_address
;
492 unsigned int len
= sg
->dma_length
;
493 unsigned long npages
, entry
;
497 npages
= iommu_num_pages(dma_handle
, len
);
498 iommu_range_free(iommu
, dma_handle
, npages
);
500 entry
= ((dma_handle
- iommu
->page_table_map_base
) >> IO_PAGE_SHIFT
);
504 num
= pci_sun4v_iommu_demap(devhandle
, HV_PCI_TSBID(0, entry
),
513 spin_unlock_irqrestore(&iommu
->lock
, flags
);
516 static void dma_4v_sync_single_for_cpu(struct device
*dev
,
517 dma_addr_t bus_addr
, size_t sz
,
518 enum dma_data_direction direction
)
520 /* Nothing to do... */
523 static void dma_4v_sync_sg_for_cpu(struct device
*dev
,
524 struct scatterlist
*sglist
, int nelems
,
525 enum dma_data_direction direction
)
527 /* Nothing to do... */
530 const struct dma_ops sun4v_dma_ops
= {
531 .alloc_coherent
= dma_4v_alloc_coherent
,
532 .free_coherent
= dma_4v_free_coherent
,
533 .map_single
= dma_4v_map_single
,
534 .unmap_single
= dma_4v_unmap_single
,
535 .map_sg
= dma_4v_map_sg
,
536 .unmap_sg
= dma_4v_unmap_sg
,
537 .sync_single_for_cpu
= dma_4v_sync_single_for_cpu
,
538 .sync_sg_for_cpu
= dma_4v_sync_sg_for_cpu
,
541 static void __init
pci_sun4v_scan_bus(struct pci_pbm_info
*pbm
)
543 struct property
*prop
;
544 struct device_node
*dp
;
547 prop
= of_find_property(dp
, "66mhz-capable", NULL
);
548 pbm
->is_66mhz_capable
= (prop
!= NULL
);
549 pbm
->pci_bus
= pci_scan_one_pbm(pbm
);
551 /* XXX register error interrupt handlers XXX */
554 static unsigned long __init
probe_existing_entries(struct pci_pbm_info
*pbm
,
557 struct iommu_arena
*arena
= &iommu
->arena
;
558 unsigned long i
, cnt
= 0;
561 devhandle
= pbm
->devhandle
;
562 for (i
= 0; i
< arena
->limit
; i
++) {
563 unsigned long ret
, io_attrs
, ra
;
565 ret
= pci_sun4v_iommu_getmap(devhandle
,
569 if (page_in_phys_avail(ra
)) {
570 pci_sun4v_iommu_demap(devhandle
,
571 HV_PCI_TSBID(0, i
), 1);
574 __set_bit(i
, arena
->map
);
582 static void __init
pci_sun4v_iommu_init(struct pci_pbm_info
*pbm
)
584 struct iommu
*iommu
= pbm
->iommu
;
585 struct property
*prop
;
586 unsigned long num_tsb_entries
, sz
, tsbsize
;
587 u32 vdma
[2], dma_mask
, dma_offset
;
589 prop
= of_find_property(pbm
->prom_node
, "virtual-dma", NULL
);
591 u32
*val
= prop
->value
;
596 /* No property, use default values. */
597 vdma
[0] = 0x80000000;
598 vdma
[1] = 0x80000000;
601 if ((vdma
[0] | vdma
[1]) & ~IO_PAGE_MASK
) {
602 prom_printf("PCI-SUN4V: strange virtual-dma[%08x:%08x].\n",
607 dma_mask
= (roundup_pow_of_two(vdma
[1]) - 1UL);
608 num_tsb_entries
= vdma
[1] / IO_PAGE_SIZE
;
609 tsbsize
= num_tsb_entries
* sizeof(iopte_t
);
611 dma_offset
= vdma
[0];
613 /* Setup initial software IOMMU state. */
614 spin_lock_init(&iommu
->lock
);
615 iommu
->ctx_lowest_free
= 1;
616 iommu
->page_table_map_base
= dma_offset
;
617 iommu
->dma_addr_mask
= dma_mask
;
619 /* Allocate and initialize the free area map. */
620 sz
= (num_tsb_entries
+ 7) / 8;
621 sz
= (sz
+ 7UL) & ~7UL;
622 iommu
->arena
.map
= kzalloc(sz
, GFP_KERNEL
);
623 if (!iommu
->arena
.map
) {
624 prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n");
627 iommu
->arena
.limit
= num_tsb_entries
;
629 sz
= probe_existing_entries(pbm
, iommu
);
631 printk("%s: Imported %lu TSB entries from OBP\n",
635 #ifdef CONFIG_PCI_MSI
636 struct pci_sun4v_msiq_entry
{
638 #define MSIQ_VERSION_MASK 0xffffffff00000000UL
639 #define MSIQ_VERSION_SHIFT 32
640 #define MSIQ_TYPE_MASK 0x00000000000000ffUL
641 #define MSIQ_TYPE_SHIFT 0
642 #define MSIQ_TYPE_NONE 0x00
643 #define MSIQ_TYPE_MSG 0x01
644 #define MSIQ_TYPE_MSI32 0x02
645 #define MSIQ_TYPE_MSI64 0x03
646 #define MSIQ_TYPE_INTX 0x08
647 #define MSIQ_TYPE_NONE2 0xff
652 u64 req_id
; /* bus/device/func */
653 #define MSIQ_REQID_BUS_MASK 0xff00UL
654 #define MSIQ_REQID_BUS_SHIFT 8
655 #define MSIQ_REQID_DEVICE_MASK 0x00f8UL
656 #define MSIQ_REQID_DEVICE_SHIFT 3
657 #define MSIQ_REQID_FUNC_MASK 0x0007UL
658 #define MSIQ_REQID_FUNC_SHIFT 0
662 /* The format of this value is message type dependent.
663 * For MSI bits 15:0 are the data from the MSI packet.
664 * For MSI-X bits 31:0 are the data from the MSI packet.
665 * For MSG, the message code and message routing code where:
666 * bits 39:32 is the bus/device/fn of the msg target-id
667 * bits 18:16 is the message routing code
668 * bits 7:0 is the message code
669 * For INTx the low order 2-bits are:
680 static int pci_sun4v_get_head(struct pci_pbm_info
*pbm
, unsigned long msiqid
,
683 unsigned long err
, limit
;
685 err
= pci_sun4v_msiq_gethead(pbm
->devhandle
, msiqid
, head
);
689 limit
= pbm
->msiq_ent_count
* sizeof(struct pci_sun4v_msiq_entry
);
690 if (unlikely(*head
>= limit
))
696 static int pci_sun4v_dequeue_msi(struct pci_pbm_info
*pbm
,
697 unsigned long msiqid
, unsigned long *head
,
700 struct pci_sun4v_msiq_entry
*ep
;
701 unsigned long err
, type
;
703 /* Note: void pointer arithmetic, 'head' is a byte offset */
704 ep
= (pbm
->msi_queues
+ ((msiqid
- pbm
->msiq_first
) *
705 (pbm
->msiq_ent_count
*
706 sizeof(struct pci_sun4v_msiq_entry
))) +
709 if ((ep
->version_type
& MSIQ_TYPE_MASK
) == 0)
712 type
= (ep
->version_type
& MSIQ_TYPE_MASK
) >> MSIQ_TYPE_SHIFT
;
713 if (unlikely(type
!= MSIQ_TYPE_MSI32
&&
714 type
!= MSIQ_TYPE_MSI64
))
719 err
= pci_sun4v_msi_setstate(pbm
->devhandle
,
720 ep
->msi_data
/* msi_num */,
725 /* Clear the entry. */
726 ep
->version_type
&= ~MSIQ_TYPE_MASK
;
728 (*head
) += sizeof(struct pci_sun4v_msiq_entry
);
730 (pbm
->msiq_ent_count
* sizeof(struct pci_sun4v_msiq_entry
)))
736 static int pci_sun4v_set_head(struct pci_pbm_info
*pbm
, unsigned long msiqid
,
741 err
= pci_sun4v_msiq_sethead(pbm
->devhandle
, msiqid
, head
);
748 static int pci_sun4v_msi_setup(struct pci_pbm_info
*pbm
, unsigned long msiqid
,
749 unsigned long msi
, int is_msi64
)
751 if (pci_sun4v_msi_setmsiq(pbm
->devhandle
, msi
, msiqid
,
753 HV_MSITYPE_MSI64
: HV_MSITYPE_MSI32
)))
755 if (pci_sun4v_msi_setstate(pbm
->devhandle
, msi
, HV_MSISTATE_IDLE
))
757 if (pci_sun4v_msi_setvalid(pbm
->devhandle
, msi
, HV_MSIVALID_VALID
))
762 static int pci_sun4v_msi_teardown(struct pci_pbm_info
*pbm
, unsigned long msi
)
764 unsigned long err
, msiqid
;
766 err
= pci_sun4v_msi_getmsiq(pbm
->devhandle
, msi
, &msiqid
);
770 pci_sun4v_msi_setvalid(pbm
->devhandle
, msi
, HV_MSIVALID_INVALID
);
775 static int pci_sun4v_msiq_alloc(struct pci_pbm_info
*pbm
)
777 unsigned long q_size
, alloc_size
, pages
, order
;
780 q_size
= pbm
->msiq_ent_count
* sizeof(struct pci_sun4v_msiq_entry
);
781 alloc_size
= (pbm
->msiq_num
* q_size
);
782 order
= get_order(alloc_size
);
783 pages
= __get_free_pages(GFP_KERNEL
| __GFP_COMP
, order
);
785 printk(KERN_ERR
"MSI: Cannot allocate MSI queues (o=%lu).\n",
789 memset((char *)pages
, 0, PAGE_SIZE
<< order
);
790 pbm
->msi_queues
= (void *) pages
;
792 for (i
= 0; i
< pbm
->msiq_num
; i
++) {
793 unsigned long err
, base
= __pa(pages
+ (i
* q_size
));
794 unsigned long ret1
, ret2
;
796 err
= pci_sun4v_msiq_conf(pbm
->devhandle
,
798 base
, pbm
->msiq_ent_count
);
800 printk(KERN_ERR
"MSI: msiq register fails (err=%lu)\n",
805 err
= pci_sun4v_msiq_info(pbm
->devhandle
,
809 printk(KERN_ERR
"MSI: Cannot read msiq (err=%lu)\n",
813 if (ret1
!= base
|| ret2
!= pbm
->msiq_ent_count
) {
814 printk(KERN_ERR
"MSI: Bogus qconf "
815 "expected[%lx:%x] got[%lx:%lx]\n",
816 base
, pbm
->msiq_ent_count
,
825 free_pages(pages
, order
);
829 static void pci_sun4v_msiq_free(struct pci_pbm_info
*pbm
)
831 unsigned long q_size
, alloc_size
, pages
, order
;
834 for (i
= 0; i
< pbm
->msiq_num
; i
++) {
835 unsigned long msiqid
= pbm
->msiq_first
+ i
;
837 (void) pci_sun4v_msiq_conf(pbm
->devhandle
, msiqid
, 0UL, 0);
840 q_size
= pbm
->msiq_ent_count
* sizeof(struct pci_sun4v_msiq_entry
);
841 alloc_size
= (pbm
->msiq_num
* q_size
);
842 order
= get_order(alloc_size
);
844 pages
= (unsigned long) pbm
->msi_queues
;
846 free_pages(pages
, order
);
848 pbm
->msi_queues
= NULL
;
851 static int pci_sun4v_msiq_build_irq(struct pci_pbm_info
*pbm
,
852 unsigned long msiqid
,
853 unsigned long devino
)
855 unsigned int virt_irq
= sun4v_build_irq(pbm
->devhandle
, devino
);
860 if (pci_sun4v_msiq_setstate(pbm
->devhandle
, msiqid
, HV_MSIQSTATE_IDLE
))
862 if (pci_sun4v_msiq_setvalid(pbm
->devhandle
, msiqid
, HV_MSIQ_VALID
))
868 static const struct sparc64_msiq_ops pci_sun4v_msiq_ops
= {
869 .get_head
= pci_sun4v_get_head
,
870 .dequeue_msi
= pci_sun4v_dequeue_msi
,
871 .set_head
= pci_sun4v_set_head
,
872 .msi_setup
= pci_sun4v_msi_setup
,
873 .msi_teardown
= pci_sun4v_msi_teardown
,
874 .msiq_alloc
= pci_sun4v_msiq_alloc
,
875 .msiq_free
= pci_sun4v_msiq_free
,
876 .msiq_build_irq
= pci_sun4v_msiq_build_irq
,
879 static void pci_sun4v_msi_init(struct pci_pbm_info
*pbm
)
881 sparc64_pbm_msi_init(pbm
, &pci_sun4v_msiq_ops
);
883 #else /* CONFIG_PCI_MSI */
884 static void pci_sun4v_msi_init(struct pci_pbm_info
*pbm
)
887 #endif /* !(CONFIG_PCI_MSI) */
889 static void __init
pci_sun4v_pbm_init(struct pci_controller_info
*p
,
890 struct device_node
*dp
, u32 devhandle
)
892 struct pci_pbm_info
*pbm
;
894 if (devhandle
& 0x40)
899 pbm
->next
= pci_pbm_root
;
902 pbm
->scan_bus
= pci_sun4v_scan_bus
;
903 pbm
->pci_ops
= &sun4v_pci_ops
;
904 pbm
->config_space_reg_bits
= 12;
906 pbm
->index
= pci_num_pbms
++;
911 pbm
->devhandle
= devhandle
;
913 pbm
->name
= dp
->full_name
;
915 printk("%s: SUN4V PCI Bus Module\n", pbm
->name
);
917 pci_determine_mem_io_space(pbm
);
919 pci_get_pbm_props(pbm
);
920 pci_sun4v_iommu_init(pbm
);
921 pci_sun4v_msi_init(pbm
);
924 void __init
sun4v_pci_init(struct device_node
*dp
, char *model_name
)
926 static int hvapi_negotiated
= 0;
927 struct pci_controller_info
*p
;
928 struct pci_pbm_info
*pbm
;
930 struct property
*prop
;
931 struct linux_prom64_registers
*regs
;
935 if (!hvapi_negotiated
++) {
936 int err
= sun4v_hvapi_register(HV_GRP_PCI
,
941 prom_printf("SUN4V_PCI: Could not register hvapi, "
945 printk("SUN4V_PCI: Registered hvapi major[%lu] minor[%lu]\n",
946 vpci_major
, vpci_minor
);
948 dma_ops
= &sun4v_dma_ops
;
951 prop
= of_find_property(dp
, "reg", NULL
);
953 prom_printf("SUN4V_PCI: Could not find config registers\n");
958 devhandle
= (regs
->phys_addr
>> 32UL) & 0x0fffffff;
960 for (pbm
= pci_pbm_root
; pbm
; pbm
= pbm
->next
) {
961 if (pbm
->devhandle
== (devhandle
^ 0x40)) {
962 pci_sun4v_pbm_init(pbm
->parent
, dp
, devhandle
);
967 for_each_possible_cpu(i
) {
968 unsigned long page
= get_zeroed_page(GFP_ATOMIC
);
971 goto fatal_memory_error
;
973 per_cpu(iommu_batch
, i
).pglist
= (u64
*) page
;
976 p
= kzalloc(sizeof(struct pci_controller_info
), GFP_ATOMIC
);
978 goto fatal_memory_error
;
980 iommu
= kzalloc(sizeof(struct iommu
), GFP_ATOMIC
);
982 goto fatal_memory_error
;
984 p
->pbm_A
.iommu
= iommu
;
986 iommu
= kzalloc(sizeof(struct iommu
), GFP_ATOMIC
);
988 goto fatal_memory_error
;
990 p
->pbm_B
.iommu
= iommu
;
992 pci_sun4v_pbm_init(p
, dp
, devhandle
);
996 prom_printf("SUN4V_PCI: Fatal memory allocation error.\n");