Commit | Line | Data |
---|---|---|
c24a8a7a J |
1 | /* |
2 | * Copyright (c) 2003-2012 Broadcom Corporation | |
3 | * All Rights Reserved | |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the Broadcom | |
9 | * license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or without | |
12 | * modification, are permitted provided that the following conditions | |
13 | * are met: | |
14 | * | |
15 | * 1. Redistributions of source code must retain the above copyright | |
16 | * notice, this list of conditions and the following disclaimer. | |
17 | * 2. Redistributions in binary form must reproduce the above copyright | |
18 | * notice, this list of conditions and the following disclaimer in | |
19 | * the documentation and/or other materials provided with the | |
20 | * distribution. | |
21 | * | |
22 | * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR | |
23 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED | |
24 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
25 | * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE | |
26 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
27 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
28 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
29 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
30 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
31 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
32 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
33 | */ | |
34 | ||
35 | #include <linux/types.h> | |
36 | #include <linux/pci.h> | |
37 | #include <linux/kernel.h> | |
38 | #include <linux/init.h> | |
39 | #include <linux/msi.h> | |
40 | #include <linux/mm.h> | |
41 | #include <linux/irq.h> | |
42 | #include <linux/irqdesc.h> | |
43 | #include <linux/console.h> | |
44 | ||
45 | #include <asm/io.h> | |
46 | ||
47 | #include <asm/netlogic/interrupt.h> | |
48 | #include <asm/netlogic/haldefs.h> | |
49 | #include <asm/netlogic/common.h> | |
50 | #include <asm/netlogic/mips-extns.h> | |
51 | ||
52 | #include <asm/netlogic/xlp-hal/iomap.h> | |
53 | #include <asm/netlogic/xlp-hal/xlp.h> | |
54 | #include <asm/netlogic/xlp-hal/pic.h> | |
55 | #include <asm/netlogic/xlp-hal/pcibus.h> | |
56 | #include <asm/netlogic/xlp-hal/bridge.h> | |
57 | ||
58 | #define XLP_MSIVEC_PER_LINK 32 | |
59 | #define XLP_MSIXVEC_TOTAL 32 | |
60 | #define XLP_MSIXVEC_PER_LINK 8 | |
61 | ||
62 | /* 128 MSI irqs per node, mapped starting at NLM_MSI_VEC_BASE */ | |
63 | static inline int nlm_link_msiirq(int link, int msivec) | |
64 | { | |
65 | return NLM_MSI_VEC_BASE + link * XLP_MSIVEC_PER_LINK + msivec; | |
66 | } | |
67 | ||
68 | static inline int nlm_irq_msivec(int irq) | |
69 | { | |
70 | return irq % XLP_MSIVEC_PER_LINK; | |
71 | } | |
72 | ||
73 | static inline int nlm_irq_msilink(int irq) | |
74 | { | |
75 | return (irq % (XLP_MSIVEC_PER_LINK * PCIE_NLINKS)) / | |
76 | XLP_MSIVEC_PER_LINK; | |
77 | } | |
78 | ||
79 | /* | |
80 | * Only 32 MSI-X vectors are possible because there are only 32 PIC | |
81 | * interrupts for MSI. We split them statically and use 8 MSI-X vectors | |
82 | * per link - this keeps the allocation and lookup simple. | |
83 | */ | |
84 | static inline int nlm_link_msixirq(int link, int bit) | |
85 | { | |
86 | return NLM_MSIX_VEC_BASE + link * XLP_MSIXVEC_PER_LINK + bit; | |
87 | } | |
88 | ||
89 | static inline int nlm_irq_msixvec(int irq) | |
90 | { | |
91 | return irq % XLP_MSIXVEC_TOTAL; /* works when given xirq */ | |
92 | } | |
93 | ||
94 | static inline int nlm_irq_msixlink(int irq) | |
95 | { | |
96 | return nlm_irq_msixvec(irq) / XLP_MSIXVEC_PER_LINK; | |
97 | } | |
98 | ||
99 | /* | |
100 | * Per link MSI and MSI-X information, set as IRQ handler data for | |
101 | * MSI and MSI-X interrupts. | |
102 | */ | |
103 | struct xlp_msi_data { | |
104 | struct nlm_soc_info *node; | |
105 | uint64_t lnkbase; | |
106 | uint32_t msi_enabled_mask; | |
107 | uint32_t msi_alloc_mask; | |
108 | uint32_t msix_alloc_mask; | |
109 | spinlock_t msi_lock; | |
110 | }; | |
111 | ||
112 | /* | |
113 | * MSI Chip definitions | |
114 | * | |
115 | * On XLP, there is a PIC interrupt associated with each PCIe link on the | |
116 | * chip (which appears as a PCI bridge to us). This gives us 32 MSI irqa | |
117 | * per link and 128 overall. | |
118 | * | |
119 | * When a device connected to the link raises a MSI interrupt, we get a | |
120 | * link interrupt and we then have to look at PCIE_MSI_STATUS register at | |
121 | * the bridge to map it to the IRQ | |
122 | */ | |
123 | static void xlp_msi_enable(struct irq_data *d) | |
124 | { | |
125 | struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); | |
126 | unsigned long flags; | |
127 | int vec; | |
128 | ||
129 | vec = nlm_irq_msivec(d->irq); | |
130 | spin_lock_irqsave(&md->msi_lock, flags); | |
131 | md->msi_enabled_mask |= 1u << vec; | |
132 | nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask); | |
133 | spin_unlock_irqrestore(&md->msi_lock, flags); | |
134 | } | |
135 | ||
136 | static void xlp_msi_disable(struct irq_data *d) | |
137 | { | |
138 | struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); | |
139 | unsigned long flags; | |
140 | int vec; | |
141 | ||
142 | vec = nlm_irq_msivec(d->irq); | |
143 | spin_lock_irqsave(&md->msi_lock, flags); | |
144 | md->msi_enabled_mask &= ~(1u << vec); | |
145 | nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask); | |
146 | spin_unlock_irqrestore(&md->msi_lock, flags); | |
147 | } | |
148 | ||
149 | static void xlp_msi_mask_ack(struct irq_data *d) | |
150 | { | |
151 | struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); | |
152 | int link, vec; | |
153 | ||
154 | link = nlm_irq_msilink(d->irq); | |
155 | vec = nlm_irq_msivec(d->irq); | |
156 | xlp_msi_disable(d); | |
157 | ||
158 | /* Ack MSI on bridge */ | |
159 | nlm_write_reg(md->lnkbase, PCIE_MSI_STATUS, 1u << vec); | |
160 | ||
161 | /* Ack at eirr and PIC */ | |
162 | ack_c0_eirr(PIC_PCIE_LINK_MSI_IRQ(link)); | |
163 | nlm_pic_ack(md->node->picbase, PIC_IRT_PCIE_LINK_INDEX(link)); | |
164 | } | |
165 | ||
166 | static struct irq_chip xlp_msi_chip = { | |
167 | .name = "XLP-MSI", | |
168 | .irq_enable = xlp_msi_enable, | |
169 | .irq_disable = xlp_msi_disable, | |
170 | .irq_mask_ack = xlp_msi_mask_ack, | |
171 | .irq_unmask = xlp_msi_enable, | |
172 | }; | |
173 | ||
174 | /* | |
175 | * The MSI-X interrupt handling is different from MSI, there are 32 | |
176 | * MSI-X interrupts generated by the PIC and each of these correspond | |
177 | * to a MSI-X vector (0-31) that can be assigned. | |
178 | * | |
179 | * We divide the MSI-X vectors to 8 per link and do a per-link | |
180 | * allocation | |
181 | * | |
182 | * Enable and disable done using standard MSI functions. | |
183 | */ | |
184 | static void xlp_msix_mask_ack(struct irq_data *d) | |
185 | { | |
186 | struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); | |
187 | int link, msixvec; | |
188 | ||
189 | msixvec = nlm_irq_msixvec(d->irq); | |
190 | link = nlm_irq_msixlink(d->irq); | |
191 | mask_msi_irq(d); | |
192 | ||
193 | /* Ack MSI on bridge */ | |
194 | nlm_write_reg(md->lnkbase, PCIE_MSIX_STATUS, 1u << msixvec); | |
195 | ||
196 | /* Ack at eirr and PIC */ | |
197 | ack_c0_eirr(PIC_PCIE_MSIX_IRQ(link)); | |
198 | nlm_pic_ack(md->node->picbase, PIC_IRT_PCIE_MSIX_INDEX(msixvec)); | |
199 | } | |
200 | ||
201 | static struct irq_chip xlp_msix_chip = { | |
202 | .name = "XLP-MSIX", | |
203 | .irq_enable = unmask_msi_irq, | |
204 | .irq_disable = mask_msi_irq, | |
205 | .irq_mask_ack = xlp_msix_mask_ack, | |
206 | .irq_unmask = unmask_msi_irq, | |
207 | }; | |
208 | ||
209 | void destroy_irq(unsigned int irq) | |
210 | { | |
211 | /* nothing to do yet */ | |
212 | } | |
213 | ||
214 | void arch_teardown_msi_irq(unsigned int irq) | |
215 | { | |
216 | destroy_irq(irq); | |
217 | } | |
218 | ||
219 | /* | |
220 | * Setup a PCIe link for MSI. By default, the links are in | |
221 | * legacy interrupt mode. We will switch them to MSI mode | |
222 | * at the first MSI request. | |
223 | */ | |
224 | static void xlp_config_link_msi(uint64_t lnkbase, int lirq, uint64_t msiaddr) | |
225 | { | |
226 | u32 val; | |
227 | ||
228 | val = nlm_read_reg(lnkbase, PCIE_INT_EN0); | |
229 | if ((val & 0x200) == 0) { | |
230 | val |= 0x200; /* MSI Interrupt enable */ | |
231 | nlm_write_reg(lnkbase, PCIE_INT_EN0, val); | |
232 | } | |
233 | ||
234 | val = nlm_read_reg(lnkbase, 0x1); /* CMD */ | |
235 | if ((val & 0x0400) == 0) { | |
236 | val |= 0x0400; | |
237 | nlm_write_reg(lnkbase, 0x1, val); | |
238 | } | |
239 | ||
240 | /* Update IRQ in the PCI irq reg */ | |
241 | val = nlm_read_pci_reg(lnkbase, 0xf); | |
242 | val &= ~0x1fu; | |
243 | val |= (1 << 8) | lirq; | |
244 | nlm_write_pci_reg(lnkbase, 0xf, val); | |
245 | ||
246 | /* MSI addr */ | |
247 | nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRH, msiaddr >> 32); | |
248 | nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRL, msiaddr & 0xffffffff); | |
249 | ||
250 | /* MSI cap for bridge */ | |
251 | val = nlm_read_reg(lnkbase, PCIE_BRIDGE_MSI_CAP); | |
252 | if ((val & (1 << 16)) == 0) { | |
253 | val |= 0xb << 16; /* mmc32, msi enable */ | |
254 | nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_CAP, val); | |
255 | } | |
256 | } | |
257 | ||
258 | /* | |
259 | * Allocate a MSI vector on a link | |
260 | */ | |
261 | static int xlp_setup_msi(uint64_t lnkbase, int node, int link, | |
262 | struct msi_desc *desc) | |
263 | { | |
264 | struct xlp_msi_data *md; | |
265 | struct msi_msg msg; | |
266 | unsigned long flags; | |
267 | int msivec, irt, lirq, xirq, ret; | |
268 | uint64_t msiaddr; | |
269 | ||
270 | /* Get MSI data for the link */ | |
271 | lirq = PIC_PCIE_LINK_MSI_IRQ(link); | |
272 | xirq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0)); | |
273 | md = irq_get_handler_data(xirq); | |
274 | msiaddr = MSI_LINK_ADDR(node, link); | |
275 | ||
276 | spin_lock_irqsave(&md->msi_lock, flags); | |
277 | if (md->msi_alloc_mask == 0) { | |
278 | /* switch the link IRQ to MSI range */ | |
279 | xlp_config_link_msi(lnkbase, lirq, msiaddr); | |
280 | irt = PIC_IRT_PCIE_LINK_INDEX(link); | |
281 | nlm_setup_pic_irq(node, lirq, lirq, irt); | |
282 | nlm_pic_init_irt(nlm_get_node(node)->picbase, irt, lirq, | |
283 | node * NLM_CPUS_PER_NODE, 1 /*en */); | |
284 | } | |
285 | ||
286 | /* allocate a MSI vec, and tell the bridge about it */ | |
287 | msivec = fls(md->msi_alloc_mask); | |
288 | if (msivec == XLP_MSIVEC_PER_LINK) { | |
289 | spin_unlock_irqrestore(&md->msi_lock, flags); | |
290 | return -ENOMEM; | |
291 | } | |
292 | md->msi_alloc_mask |= (1u << msivec); | |
293 | spin_unlock_irqrestore(&md->msi_lock, flags); | |
294 | ||
295 | msg.address_hi = msiaddr >> 32; | |
296 | msg.address_lo = msiaddr & 0xffffffff; | |
297 | msg.data = 0xc00 | msivec; | |
298 | ||
299 | xirq = xirq + msivec; /* msi mapped to global irq space */ | |
300 | ret = irq_set_msi_desc(xirq, desc); | |
301 | if (ret < 0) { | |
302 | destroy_irq(xirq); | |
303 | return ret; | |
304 | } | |
305 | ||
306 | write_msi_msg(xirq, &msg); | |
307 | return 0; | |
308 | } | |
309 | ||
310 | /* | |
311 | * Switch a link to MSI-X mode | |
312 | */ | |
313 | static void xlp_config_link_msix(uint64_t lnkbase, int lirq, uint64_t msixaddr) | |
314 | { | |
315 | u32 val; | |
316 | ||
317 | val = nlm_read_reg(lnkbase, 0x2C); | |
318 | if ((val & 0x80000000U) == 0) { | |
319 | val |= 0x80000000U; | |
320 | nlm_write_reg(lnkbase, 0x2C, val); | |
321 | } | |
322 | val = nlm_read_reg(lnkbase, PCIE_INT_EN0); | |
323 | if ((val & 0x200) == 0) { | |
324 | val |= 0x200; /* MSI Interrupt enable */ | |
325 | nlm_write_reg(lnkbase, PCIE_INT_EN0, val); | |
326 | } | |
327 | ||
328 | val = nlm_read_reg(lnkbase, 0x1); /* CMD */ | |
329 | if ((val & 0x0400) == 0) { | |
330 | val |= 0x0400; | |
331 | nlm_write_reg(lnkbase, 0x1, val); | |
332 | } | |
333 | ||
334 | /* Update IRQ in the PCI irq reg */ | |
335 | val = nlm_read_pci_reg(lnkbase, 0xf); | |
336 | val &= ~0x1fu; | |
337 | val |= (1 << 8) | lirq; | |
338 | nlm_write_pci_reg(lnkbase, 0xf, val); | |
339 | ||
340 | /* MSI-X addresses */ | |
341 | nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_BASE, msixaddr >> 8); | |
342 | nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_LIMIT, | |
343 | (msixaddr + MSI_ADDR_SZ) >> 8); | |
344 | } | |
345 | ||
346 | /* | |
347 | * Allocate a MSI-X vector | |
348 | */ | |
349 | static int xlp_setup_msix(uint64_t lnkbase, int node, int link, | |
350 | struct msi_desc *desc) | |
351 | { | |
352 | struct xlp_msi_data *md; | |
353 | struct msi_msg msg; | |
354 | unsigned long flags; | |
355 | int t, msixvec, lirq, xirq, ret; | |
356 | uint64_t msixaddr; | |
357 | ||
358 | /* Get MSI data for the link */ | |
359 | lirq = PIC_PCIE_MSIX_IRQ(link); | |
360 | xirq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0)); | |
361 | md = irq_get_handler_data(xirq); | |
362 | msixaddr = MSIX_LINK_ADDR(node, link); | |
363 | ||
364 | spin_lock_irqsave(&md->msi_lock, flags); | |
365 | /* switch the PCIe link to MSI-X mode at the first alloc */ | |
366 | if (md->msix_alloc_mask == 0) | |
367 | xlp_config_link_msix(lnkbase, lirq, msixaddr); | |
368 | ||
369 | /* allocate a MSI-X vec, and tell the bridge about it */ | |
370 | t = fls(md->msix_alloc_mask); | |
371 | if (t == XLP_MSIXVEC_PER_LINK) { | |
372 | spin_unlock_irqrestore(&md->msi_lock, flags); | |
373 | return -ENOMEM; | |
374 | } | |
375 | md->msix_alloc_mask |= (1u << t); | |
376 | spin_unlock_irqrestore(&md->msi_lock, flags); | |
377 | ||
378 | xirq += t; | |
379 | msixvec = nlm_irq_msixvec(xirq); | |
380 | msg.address_hi = msixaddr >> 32; | |
381 | msg.address_lo = msixaddr & 0xffffffff; | |
382 | msg.data = 0xc00 | msixvec; | |
383 | ||
384 | ret = irq_set_msi_desc(xirq, desc); | |
385 | if (ret < 0) { | |
386 | destroy_irq(xirq); | |
387 | return ret; | |
388 | } | |
389 | ||
390 | write_msi_msg(xirq, &msg); | |
391 | return 0; | |
392 | } | |
393 | ||
394 | int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) | |
395 | { | |
396 | struct pci_dev *lnkdev; | |
397 | uint64_t lnkbase; | |
398 | int node, link, slot; | |
399 | ||
400 | lnkdev = xlp_get_pcie_link(dev); | |
401 | if (lnkdev == NULL) { | |
402 | dev_err(&dev->dev, "Could not find bridge\n"); | |
403 | return 1; | |
404 | } | |
405 | slot = PCI_SLOT(lnkdev->devfn); | |
406 | link = PCI_FUNC(lnkdev->devfn); | |
407 | node = slot / 8; | |
408 | lnkbase = nlm_get_pcie_base(node, link); | |
409 | ||
410 | if (desc->msi_attrib.is_msix) | |
411 | return xlp_setup_msix(lnkbase, node, link, desc); | |
412 | else | |
413 | return xlp_setup_msi(lnkbase, node, link, desc); | |
414 | } | |
415 | ||
416 | void __init xlp_init_node_msi_irqs(int node, int link) | |
417 | { | |
418 | struct nlm_soc_info *nodep; | |
419 | struct xlp_msi_data *md; | |
420 | int irq, i, irt, msixvec; | |
421 | ||
422 | pr_info("[%d %d] Init node PCI IRT\n", node, link); | |
423 | nodep = nlm_get_node(node); | |
424 | ||
425 | /* Alloc an MSI block for the link */ | |
426 | md = kzalloc(sizeof(*md), GFP_KERNEL); | |
427 | spin_lock_init(&md->msi_lock); | |
428 | md->msi_enabled_mask = 0; | |
429 | md->msi_alloc_mask = 0; | |
430 | md->msix_alloc_mask = 0; | |
431 | md->node = nodep; | |
432 | md->lnkbase = nlm_get_pcie_base(node, link); | |
433 | ||
434 | /* extended space for MSI interrupts */ | |
435 | irq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0)); | |
436 | for (i = irq; i < irq + XLP_MSIVEC_PER_LINK; i++) { | |
437 | irq_set_chip_and_handler(i, &xlp_msi_chip, handle_level_irq); | |
438 | irq_set_handler_data(i, md); | |
439 | } | |
440 | ||
441 | for (i = 0; i < XLP_MSIXVEC_PER_LINK; i++) { | |
442 | /* Initialize MSI-X irts to generate one interrupt per link */ | |
443 | msixvec = link * XLP_MSIXVEC_PER_LINK + i; | |
444 | irt = PIC_IRT_PCIE_MSIX_INDEX(msixvec); | |
445 | nlm_pic_init_irt(nodep->picbase, irt, PIC_PCIE_MSIX_IRQ(link), | |
446 | node * NLM_CPUS_PER_NODE, 1 /* enable */); | |
447 | ||
448 | /* Initialize MSI-X extended irq space for the link */ | |
449 | irq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, i)); | |
450 | irq_set_chip_and_handler(irq, &xlp_msix_chip, handle_level_irq); | |
451 | irq_set_handler_data(irq, md); | |
452 | } | |
453 | ||
454 | } | |
455 | ||
456 | void nlm_dispatch_msi(int node, int lirq) | |
457 | { | |
458 | struct xlp_msi_data *md; | |
459 | int link, i, irqbase; | |
460 | u32 status; | |
461 | ||
462 | link = lirq - PIC_PCIE_LINK_MSI_IRQ_BASE; | |
463 | irqbase = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0)); | |
464 | md = irq_get_handler_data(irqbase); | |
465 | status = nlm_read_reg(md->lnkbase, PCIE_MSI_STATUS) & | |
466 | md->msi_enabled_mask; | |
467 | while (status) { | |
468 | i = __ffs(status); | |
469 | do_IRQ(irqbase + i); | |
470 | status &= status - 1; | |
471 | } | |
472 | } | |
473 | ||
474 | void nlm_dispatch_msix(int node, int lirq) | |
475 | { | |
476 | struct xlp_msi_data *md; | |
477 | int link, i, irqbase; | |
478 | u32 status; | |
479 | ||
480 | link = lirq - PIC_PCIE_MSIX_IRQ_BASE; | |
481 | irqbase = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0)); | |
482 | md = irq_get_handler_data(irqbase); | |
483 | status = nlm_read_reg(md->lnkbase, PCIE_MSIX_STATUS); | |
484 | ||
485 | /* narrow it down to the MSI-x vectors for our link */ | |
486 | status = (status >> (link * XLP_MSIXVEC_PER_LINK)) & | |
487 | ((1 << XLP_MSIXVEC_PER_LINK) - 1); | |
488 | ||
489 | while (status) { | |
490 | i = __ffs(status); | |
491 | do_IRQ(irqbase + i); | |
492 | status &= status - 1; | |
493 | } | |
494 | } |