Commit | Line | Data |
---|---|---|
e8635b48 DD |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
1aa2b278 | 6 | * Copyright (C) 2005-2009, 2010 Cavium Networks |
e8635b48 DD |
7 | */ |
8 | #include <linux/kernel.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/msi.h> | |
11 | #include <linux/spinlock.h> | |
12 | #include <linux/interrupt.h> | |
13 | ||
14 | #include <asm/octeon/octeon.h> | |
15 | #include <asm/octeon/cvmx-npi-defs.h> | |
16 | #include <asm/octeon/cvmx-pci-defs.h> | |
17 | #include <asm/octeon/cvmx-npei-defs.h> | |
d19648d7 | 18 | #include <asm/octeon/cvmx-sli-defs.h> |
e8635b48 | 19 | #include <asm/octeon/cvmx-pexp-defs.h> |
01a6221a | 20 | #include <asm/octeon/pci-octeon.h> |
e8635b48 DD |
21 | |
22 | /* | |
23 | * Each bit in msi_free_irq_bitmask represents a MSI interrupt that is | |
24 | * in use. | |
25 | */ | |
1aa2b278 | 26 | static u64 msi_free_irq_bitmask[4]; |
e8635b48 DD |
27 | |
28 | /* | |
29 | * Each bit in msi_multiple_irq_bitmask tells that the device using | |
30 | * this bit in msi_free_irq_bitmask is also using the next bit. This | |
31 | * is used so we can disable all of the MSI interrupts when a device | |
32 | * uses multiple. | |
33 | */ | |
1aa2b278 | 34 | static u64 msi_multiple_irq_bitmask[4]; |
e8635b48 DD |
35 | |
36 | /* | |
37 | * This lock controls updates to msi_free_irq_bitmask and | |
38 | * msi_multiple_irq_bitmask. | |
39 | */ | |
40 | static DEFINE_SPINLOCK(msi_free_irq_bitmask_lock); | |
41 | ||
1aa2b278 DD |
42 | /* |
43 | * Number of MSI IRQs used. This variable is set up in | |
44 | * the module init time. | |
45 | */ | |
46 | static int msi_irq_size; | |
e8635b48 DD |
47 | |
48 | /** | |
49 | * Called when a driver request MSI interrupts instead of the | |
50 | * legacy INT A-D. This routine will allocate multiple interrupts | |
51 | * for MSI devices that support them. A device can override this by | |
52 | * programming the MSI control bits [6:4] before calling | |
53 | * pci_enable_msi(). | |
54 | * | |
01a6221a DD |
55 | * @dev: Device requesting MSI interrupts |
56 | * @desc: MSI descriptor | |
e8635b48 DD |
57 | * |
58 | * Returns 0 on success. | |
59 | */ | |
60 | int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) | |
61 | { | |
62 | struct msi_msg msg; | |
1aa2b278 | 63 | u16 control; |
e8635b48 DD |
64 | int configured_private_bits; |
65 | int request_private_bits; | |
1aa2b278 | 66 | int irq = 0; |
e8635b48 | 67 | int irq_step; |
1aa2b278 DD |
68 | u64 search_mask; |
69 | int index; | |
e8635b48 DD |
70 | |
71 | /* | |
72 | * Read the MSI config to figure out how many IRQs this device | |
73 | * wants. Most devices only want 1, which will give | |
74 | * configured_private_bits and request_private_bits equal 0. | |
75 | */ | |
48c3c38f | 76 | pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); |
e8635b48 DD |
77 | |
78 | /* | |
79 | * If the number of private bits has been configured then use | |
80 | * that value instead of the requested number. This gives the | |
81 | * driver the chance to override the number of interrupts | |
82 | * before calling pci_enable_msi(). | |
83 | */ | |
84 | configured_private_bits = (control & PCI_MSI_FLAGS_QSIZE) >> 4; | |
85 | if (configured_private_bits == 0) { | |
86 | /* Nothing is configured, so use the hardware requested size */ | |
87 | request_private_bits = (control & PCI_MSI_FLAGS_QMASK) >> 1; | |
88 | } else { | |
89 | /* | |
90 | * Use the number of configured bits, assuming the | |
91 | * driver wanted to override the hardware request | |
92 | * value. | |
93 | */ | |
94 | request_private_bits = configured_private_bits; | |
95 | } | |
96 | ||
97 | /* | |
98 | * The PCI 2.3 spec mandates that there are at most 32 | |
99 | * interrupts. If this device asks for more, only give it one. | |
100 | */ | |
101 | if (request_private_bits > 5) | |
102 | request_private_bits = 0; | |
103 | ||
104 | try_only_one: | |
105 | /* | |
106 | * The IRQs have to be aligned on a power of two based on the | |
107 | * number being requested. | |
108 | */ | |
109 | irq_step = 1 << request_private_bits; | |
110 | ||
111 | /* Mask with one bit for each IRQ */ | |
112 | search_mask = (1 << irq_step) - 1; | |
113 | ||
114 | /* | |
115 | * We're going to search msi_free_irq_bitmask_lock for zero | |
116 | * bits. This represents an MSI interrupt number that isn't in | |
117 | * use. | |
118 | */ | |
119 | spin_lock(&msi_free_irq_bitmask_lock); | |
1aa2b278 DD |
120 | for (index = 0; index < msi_irq_size/64; index++) { |
121 | for (irq = 0; irq < 64; irq += irq_step) { | |
122 | if ((msi_free_irq_bitmask[index] & (search_mask << irq)) == 0) { | |
123 | msi_free_irq_bitmask[index] |= search_mask << irq; | |
124 | msi_multiple_irq_bitmask[index] |= (search_mask >> 1) << irq; | |
125 | goto msi_irq_allocated; | |
126 | } | |
e8635b48 DD |
127 | } |
128 | } | |
1aa2b278 | 129 | msi_irq_allocated: |
e8635b48 DD |
130 | spin_unlock(&msi_free_irq_bitmask_lock); |
131 | ||
132 | /* Make sure the search for available interrupts didn't fail */ | |
133 | if (irq >= 64) { | |
134 | if (request_private_bits) { | |
1aa2b278 | 135 | pr_err("arch_setup_msi_irq: Unable to find %d free interrupts, trying just one", |
e8635b48 DD |
136 | 1 << request_private_bits); |
137 | request_private_bits = 0; | |
138 | goto try_only_one; | |
139 | } else | |
1aa2b278 | 140 | panic("arch_setup_msi_irq: Unable to find a free MSI interrupt"); |
e8635b48 DD |
141 | } |
142 | ||
143 | /* MSI interrupts start at logical IRQ OCTEON_IRQ_MSI_BIT0 */ | |
1aa2b278 | 144 | irq += index*64; |
e8635b48 DD |
145 | irq += OCTEON_IRQ_MSI_BIT0; |
146 | ||
147 | switch (octeon_dma_bar_type) { | |
148 | case OCTEON_DMA_BAR_TYPE_SMALL: | |
149 | /* When not using big bar, Bar 0 is based at 128MB */ | |
150 | msg.address_lo = | |
151 | ((128ul << 20) + CVMX_PCI_MSI_RCV) & 0xffffffff; | |
152 | msg.address_hi = ((128ul << 20) + CVMX_PCI_MSI_RCV) >> 32; | |
7f02c463 | 153 | break; |
e8635b48 DD |
154 | case OCTEON_DMA_BAR_TYPE_BIG: |
155 | /* When using big bar, Bar 0 is based at 0 */ | |
156 | msg.address_lo = (0 + CVMX_PCI_MSI_RCV) & 0xffffffff; | |
157 | msg.address_hi = (0 + CVMX_PCI_MSI_RCV) >> 32; | |
158 | break; | |
159 | case OCTEON_DMA_BAR_TYPE_PCIE: | |
160 | /* When using PCIe, Bar 0 is based at 0 */ | |
161 | /* FIXME CVMX_NPEI_MSI_RCV* other than 0? */ | |
162 | msg.address_lo = (0 + CVMX_NPEI_PCIE_MSI_RCV) & 0xffffffff; | |
163 | msg.address_hi = (0 + CVMX_NPEI_PCIE_MSI_RCV) >> 32; | |
164 | break; | |
d19648d7 ES |
165 | case OCTEON_DMA_BAR_TYPE_PCIE2: |
166 | /* When using PCIe2, Bar 0 is based at 0 */ | |
167 | msg.address_lo = (0 + CVMX_SLI_PCIE_MSI_RCV) & 0xffffffff; | |
168 | msg.address_hi = (0 + CVMX_SLI_PCIE_MSI_RCV) >> 32; | |
169 | break; | |
e8635b48 | 170 | default: |
ab75dc02 | 171 | panic("arch_setup_msi_irq: Invalid octeon_dma_bar_type"); |
e8635b48 DD |
172 | } |
173 | msg.data = irq - OCTEON_IRQ_MSI_BIT0; | |
174 | ||
175 | /* Update the number of IRQs the device has available to it */ | |
176 | control &= ~PCI_MSI_FLAGS_QSIZE; | |
177 | control |= request_private_bits << 4; | |
48c3c38f | 178 | pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); |
e8635b48 | 179 | |
e4ec7989 | 180 | irq_set_msi_desc(irq, desc); |
83a18912 | 181 | pci_write_msi_msg(irq, &msg); |
e8635b48 DD |
182 | return 0; |
183 | } | |
184 | ||
52a0f00b CC |
185 | int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) |
186 | { | |
187 | struct msi_desc *entry; | |
188 | int ret; | |
189 | ||
190 | /* | |
191 | * MSI-X is not supported. | |
192 | */ | |
193 | if (type == PCI_CAP_ID_MSIX) | |
194 | return -EINVAL; | |
195 | ||
196 | /* | |
197 | * If an architecture wants to support multiple MSI, it needs to | |
198 | * override arch_setup_msi_irqs() | |
199 | */ | |
200 | if (type == PCI_CAP_ID_MSI && nvec > 1) | |
201 | return 1; | |
202 | ||
bbcffac3 | 203 | for_each_pci_msi_entry(entry, dev) { |
52a0f00b CC |
204 | ret = arch_setup_msi_irq(dev, entry); |
205 | if (ret < 0) | |
206 | return ret; | |
207 | if (ret > 0) | |
208 | return -ENOSPC; | |
209 | } | |
210 | ||
211 | return 0; | |
212 | } | |
e8635b48 DD |
213 | |
214 | /** | |
215 | * Called when a device no longer needs its MSI interrupts. All | |
216 | * MSI interrupts for the device are freed. | |
217 | * | |
218 | * @irq: The devices first irq number. There may be multple in sequence. | |
219 | */ | |
220 | void arch_teardown_msi_irq(unsigned int irq) | |
221 | { | |
222 | int number_irqs; | |
1aa2b278 DD |
223 | u64 bitmask; |
224 | int index = 0; | |
225 | int irq0; | |
e8635b48 | 226 | |
1aa2b278 DD |
227 | if ((irq < OCTEON_IRQ_MSI_BIT0) |
228 | || (irq > msi_irq_size + OCTEON_IRQ_MSI_BIT0)) | |
e8635b48 DD |
229 | panic("arch_teardown_msi_irq: Attempted to teardown illegal " |
230 | "MSI interrupt (%d)", irq); | |
1aa2b278 | 231 | |
e8635b48 | 232 | irq -= OCTEON_IRQ_MSI_BIT0; |
1aa2b278 DD |
233 | index = irq / 64; |
234 | irq0 = irq % 64; | |
e8635b48 DD |
235 | |
236 | /* | |
237 | * Count the number of IRQs we need to free by looking at the | |
238 | * msi_multiple_irq_bitmask. Each bit set means that the next | |
239 | * IRQ is also owned by this device. | |
240 | */ | |
241 | number_irqs = 0; | |
1aa2b278 DD |
242 | while ((irq0 + number_irqs < 64) && |
243 | (msi_multiple_irq_bitmask[index] | |
244 | & (1ull << (irq0 + number_irqs)))) | |
e8635b48 DD |
245 | number_irqs++; |
246 | number_irqs++; | |
247 | /* Mask with one bit for each IRQ */ | |
248 | bitmask = (1 << number_irqs) - 1; | |
249 | /* Shift the mask to the correct bit location */ | |
1aa2b278 DD |
250 | bitmask <<= irq0; |
251 | if ((msi_free_irq_bitmask[index] & bitmask) != bitmask) | |
e8635b48 DD |
252 | panic("arch_teardown_msi_irq: Attempted to teardown MSI " |
253 | "interrupt (%d) not in use", irq); | |
254 | ||
255 | /* Checks are done, update the in use bitmask */ | |
256 | spin_lock(&msi_free_irq_bitmask_lock); | |
1aa2b278 DD |
257 | msi_free_irq_bitmask[index] &= ~bitmask; |
258 | msi_multiple_irq_bitmask[index] &= ~bitmask; | |
e8635b48 DD |
259 | spin_unlock(&msi_free_irq_bitmask_lock); |
260 | } | |
261 | ||
1aa2b278 | 262 | static DEFINE_RAW_SPINLOCK(octeon_irq_msi_lock); |
e8635b48 | 263 | |
1aa2b278 DD |
264 | static u64 msi_rcv_reg[4]; |
265 | static u64 mis_ena_reg[4]; | |
266 | ||
0c326387 | 267 | static void octeon_irq_msi_enable_pcie(struct irq_data *data) |
e8635b48 | 268 | { |
1aa2b278 DD |
269 | u64 en; |
270 | unsigned long flags; | |
0c326387 | 271 | int msi_number = data->irq - OCTEON_IRQ_MSI_BIT0; |
1aa2b278 DD |
272 | int irq_index = msi_number >> 6; |
273 | int irq_bit = msi_number & 0x3f; | |
274 | ||
275 | raw_spin_lock_irqsave(&octeon_irq_msi_lock, flags); | |
276 | en = cvmx_read_csr(mis_ena_reg[irq_index]); | |
277 | en |= 1ull << irq_bit; | |
278 | cvmx_write_csr(mis_ena_reg[irq_index], en); | |
279 | cvmx_read_csr(mis_ena_reg[irq_index]); | |
280 | raw_spin_unlock_irqrestore(&octeon_irq_msi_lock, flags); | |
281 | } | |
e8635b48 | 282 | |
0c326387 | 283 | static void octeon_irq_msi_disable_pcie(struct irq_data *data) |
1aa2b278 DD |
284 | { |
285 | u64 en; | |
286 | unsigned long flags; | |
0c326387 | 287 | int msi_number = data->irq - OCTEON_IRQ_MSI_BIT0; |
1aa2b278 DD |
288 | int irq_index = msi_number >> 6; |
289 | int irq_bit = msi_number & 0x3f; | |
290 | ||
291 | raw_spin_lock_irqsave(&octeon_irq_msi_lock, flags); | |
292 | en = cvmx_read_csr(mis_ena_reg[irq_index]); | |
293 | en &= ~(1ull << irq_bit); | |
294 | cvmx_write_csr(mis_ena_reg[irq_index], en); | |
295 | cvmx_read_csr(mis_ena_reg[irq_index]); | |
296 | raw_spin_unlock_irqrestore(&octeon_irq_msi_lock, flags); | |
e8635b48 DD |
297 | } |
298 | ||
1aa2b278 DD |
299 | static struct irq_chip octeon_irq_chip_msi_pcie = { |
300 | .name = "MSI", | |
0c326387 DD |
301 | .irq_enable = octeon_irq_msi_enable_pcie, |
302 | .irq_disable = octeon_irq_msi_disable_pcie, | |
1aa2b278 | 303 | }; |
a894f14d | 304 | |
0c326387 | 305 | static void octeon_irq_msi_enable_pci(struct irq_data *data) |
a894f14d | 306 | { |
1aa2b278 DD |
307 | /* |
308 | * Octeon PCI doesn't have the ability to mask/unmask MSI | |
309 | * interrupts individually. Instead of masking/unmasking them | |
310 | * in groups of 16, we simple assume MSI devices are well | |
311 | * behaved. MSI interrupts are always enable and the ACK is | |
312 | * assumed to be enough | |
313 | */ | |
a894f14d DD |
314 | } |
315 | ||
0c326387 | 316 | static void octeon_irq_msi_disable_pci(struct irq_data *data) |
a894f14d | 317 | { |
1aa2b278 | 318 | /* See comment in enable */ |
a894f14d DD |
319 | } |
320 | ||
1aa2b278 | 321 | static struct irq_chip octeon_irq_chip_msi_pci = { |
a894f14d | 322 | .name = "MSI", |
0c326387 DD |
323 | .irq_enable = octeon_irq_msi_enable_pci, |
324 | .irq_disable = octeon_irq_msi_disable_pci, | |
a894f14d | 325 | }; |
e8635b48 | 326 | |
01a6221a | 327 | /* |
1aa2b278 DD |
328 | * Called by the interrupt handling code when an MSI interrupt |
329 | * occurs. | |
e8635b48 | 330 | */ |
1aa2b278 | 331 | static irqreturn_t __octeon_msi_do_interrupt(int index, u64 msi_bits) |
e8635b48 | 332 | { |
a894f14d | 333 | int irq; |
1aa2b278 | 334 | int bit; |
a894f14d | 335 | |
1aa2b278 DD |
336 | bit = fls64(msi_bits); |
337 | if (bit) { | |
338 | bit--; | |
339 | /* Acknowledge it first. */ | |
340 | cvmx_write_csr(msi_rcv_reg[index], 1ull << bit); | |
341 | ||
342 | irq = bit + OCTEON_IRQ_MSI_BIT0 + 64 * index; | |
343 | do_IRQ(irq); | |
344 | return IRQ_HANDLED; | |
a894f14d | 345 | } |
1aa2b278 DD |
346 | return IRQ_NONE; |
347 | } | |
348 | ||
349 | #define OCTEON_MSI_INT_HANDLER_X(x) \ | |
350 | static irqreturn_t octeon_msi_interrupt##x(int cpl, void *dev_id) \ | |
351 | { \ | |
352 | u64 msi_bits = cvmx_read_csr(msi_rcv_reg[(x)]); \ | |
353 | return __octeon_msi_do_interrupt((x), msi_bits); \ | |
354 | } | |
355 | ||
356 | /* | |
357 | * Create octeon_msi_interrupt{0-3} function body | |
358 | */ | |
359 | OCTEON_MSI_INT_HANDLER_X(0); | |
360 | OCTEON_MSI_INT_HANDLER_X(1); | |
361 | OCTEON_MSI_INT_HANDLER_X(2); | |
362 | OCTEON_MSI_INT_HANDLER_X(3); | |
363 | ||
364 | /* | |
365 | * Initializes the MSI interrupt handling code | |
366 | */ | |
367 | int __init octeon_msi_initialize(void) | |
368 | { | |
369 | int irq; | |
370 | struct irq_chip *msi; | |
371 | ||
372 | if (octeon_dma_bar_type == OCTEON_DMA_BAR_TYPE_PCIE) { | |
373 | msi_rcv_reg[0] = CVMX_PEXP_NPEI_MSI_RCV0; | |
374 | msi_rcv_reg[1] = CVMX_PEXP_NPEI_MSI_RCV1; | |
375 | msi_rcv_reg[2] = CVMX_PEXP_NPEI_MSI_RCV2; | |
376 | msi_rcv_reg[3] = CVMX_PEXP_NPEI_MSI_RCV3; | |
377 | mis_ena_reg[0] = CVMX_PEXP_NPEI_MSI_ENB0; | |
378 | mis_ena_reg[1] = CVMX_PEXP_NPEI_MSI_ENB1; | |
379 | mis_ena_reg[2] = CVMX_PEXP_NPEI_MSI_ENB2; | |
380 | mis_ena_reg[3] = CVMX_PEXP_NPEI_MSI_ENB3; | |
381 | msi = &octeon_irq_chip_msi_pcie; | |
382 | } else { | |
383 | msi_rcv_reg[0] = CVMX_NPI_NPI_MSI_RCV; | |
384 | #define INVALID_GENERATE_ADE 0x8700000000000000ULL; | |
385 | msi_rcv_reg[1] = INVALID_GENERATE_ADE; | |
386 | msi_rcv_reg[2] = INVALID_GENERATE_ADE; | |
387 | msi_rcv_reg[3] = INVALID_GENERATE_ADE; | |
388 | mis_ena_reg[0] = INVALID_GENERATE_ADE; | |
389 | mis_ena_reg[1] = INVALID_GENERATE_ADE; | |
390 | mis_ena_reg[2] = INVALID_GENERATE_ADE; | |
391 | mis_ena_reg[3] = INVALID_GENERATE_ADE; | |
392 | msi = &octeon_irq_chip_msi_pci; | |
393 | } | |
394 | ||
395 | for (irq = OCTEON_IRQ_MSI_BIT0; irq <= OCTEON_IRQ_MSI_LAST; irq++) | |
e4ec7989 | 396 | irq_set_chip_and_handler(irq, msi, handle_simple_irq); |
a894f14d | 397 | |
e8635b48 | 398 | if (octeon_has_feature(OCTEON_FEATURE_PCIE)) { |
1aa2b278 DD |
399 | if (request_irq(OCTEON_IRQ_PCI_MSI0, octeon_msi_interrupt0, |
400 | 0, "MSI[0:63]", octeon_msi_interrupt0)) | |
01a6221a | 401 | panic("request_irq(OCTEON_IRQ_PCI_MSI0) failed"); |
1aa2b278 DD |
402 | |
403 | if (request_irq(OCTEON_IRQ_PCI_MSI1, octeon_msi_interrupt1, | |
404 | 0, "MSI[64:127]", octeon_msi_interrupt1)) | |
405 | panic("request_irq(OCTEON_IRQ_PCI_MSI1) failed"); | |
406 | ||
407 | if (request_irq(OCTEON_IRQ_PCI_MSI2, octeon_msi_interrupt2, | |
408 | 0, "MSI[127:191]", octeon_msi_interrupt2)) | |
409 | panic("request_irq(OCTEON_IRQ_PCI_MSI2) failed"); | |
410 | ||
411 | if (request_irq(OCTEON_IRQ_PCI_MSI3, octeon_msi_interrupt3, | |
412 | 0, "MSI[192:255]", octeon_msi_interrupt3)) | |
413 | panic("request_irq(OCTEON_IRQ_PCI_MSI3) failed"); | |
414 | ||
415 | msi_irq_size = 256; | |
e8635b48 | 416 | } else if (octeon_is_pci_host()) { |
1aa2b278 DD |
417 | if (request_irq(OCTEON_IRQ_PCI_MSI0, octeon_msi_interrupt0, |
418 | 0, "MSI[0:15]", octeon_msi_interrupt0)) | |
01a6221a DD |
419 | panic("request_irq(OCTEON_IRQ_PCI_MSI0) failed"); |
420 | ||
1aa2b278 DD |
421 | if (request_irq(OCTEON_IRQ_PCI_MSI1, octeon_msi_interrupt0, |
422 | 0, "MSI[16:31]", octeon_msi_interrupt0)) | |
01a6221a DD |
423 | panic("request_irq(OCTEON_IRQ_PCI_MSI1) failed"); |
424 | ||
1aa2b278 DD |
425 | if (request_irq(OCTEON_IRQ_PCI_MSI2, octeon_msi_interrupt0, |
426 | 0, "MSI[32:47]", octeon_msi_interrupt0)) | |
01a6221a DD |
427 | panic("request_irq(OCTEON_IRQ_PCI_MSI2) failed"); |
428 | ||
1aa2b278 DD |
429 | if (request_irq(OCTEON_IRQ_PCI_MSI3, octeon_msi_interrupt0, |
430 | 0, "MSI[48:63]", octeon_msi_interrupt0)) | |
01a6221a | 431 | panic("request_irq(OCTEON_IRQ_PCI_MSI3) failed"); |
1aa2b278 | 432 | msi_irq_size = 64; |
e8635b48 DD |
433 | } |
434 | return 0; | |
435 | } | |
e8635b48 | 436 | subsys_initcall(octeon_msi_initialize); |