Commit | Line | Data |
---|---|---|
c9e6bc64 ED |
1 | #ifndef _NET_GRO_CELLS_H |
2 | #define _NET_GRO_CELLS_H | |
3 | ||
4 | #include <linux/skbuff.h> | |
5 | #include <linux/slab.h> | |
6 | #include <linux/netdevice.h> | |
7 | ||
8 | struct gro_cell { | |
9 | struct sk_buff_head napi_skbs; | |
10 | struct napi_struct napi; | |
11 | } ____cacheline_aligned_in_smp; | |
12 | ||
13 | struct gro_cells { | |
14 | unsigned int gro_cells_mask; | |
15 | struct gro_cell *cells; | |
16 | }; | |
17 | ||
18 | static inline void gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) | |
19 | { | |
c9e6bc64 ED |
20 | struct gro_cell *cell = gcells->cells; |
21 | struct net_device *dev = skb->dev; | |
22 | ||
23 | if (!cell || skb_cloned(skb) || !(dev->features & NETIF_F_GRO)) { | |
24 | netif_rx(skb); | |
25 | return; | |
26 | } | |
27 | ||
28 | if (skb_rx_queue_recorded(skb)) | |
29 | cell += skb_get_rx_queue(skb) & gcells->gro_cells_mask; | |
30 | ||
31 | if (skb_queue_len(&cell->napi_skbs) > netdev_max_backlog) { | |
32 | atomic_long_inc(&dev->rx_dropped); | |
33 | kfree_skb(skb); | |
34 | return; | |
35 | } | |
36 | ||
f8e8f97c ED |
37 | /* We run in BH context */ |
38 | spin_lock(&cell->napi_skbs.lock); | |
c9e6bc64 ED |
39 | |
40 | __skb_queue_tail(&cell->napi_skbs, skb); | |
41 | if (skb_queue_len(&cell->napi_skbs) == 1) | |
42 | napi_schedule(&cell->napi); | |
43 | ||
f8e8f97c | 44 | spin_unlock(&cell->napi_skbs.lock); |
c9e6bc64 ED |
45 | } |
46 | ||
f8e8f97c | 47 | /* called unser BH context */ |
c9e6bc64 ED |
48 | static inline int gro_cell_poll(struct napi_struct *napi, int budget) |
49 | { | |
50 | struct gro_cell *cell = container_of(napi, struct gro_cell, napi); | |
51 | struct sk_buff *skb; | |
52 | int work_done = 0; | |
53 | ||
f8e8f97c | 54 | spin_lock(&cell->napi_skbs.lock); |
c9e6bc64 | 55 | while (work_done < budget) { |
f8e8f97c | 56 | skb = __skb_dequeue(&cell->napi_skbs); |
c9e6bc64 ED |
57 | if (!skb) |
58 | break; | |
f8e8f97c | 59 | spin_unlock(&cell->napi_skbs.lock); |
c9e6bc64 ED |
60 | napi_gro_receive(napi, skb); |
61 | work_done++; | |
f8e8f97c | 62 | spin_lock(&cell->napi_skbs.lock); |
c9e6bc64 ED |
63 | } |
64 | ||
65 | if (work_done < budget) | |
66 | napi_complete(napi); | |
f8e8f97c | 67 | spin_unlock(&cell->napi_skbs.lock); |
c9e6bc64 ED |
68 | return work_done; |
69 | } | |
70 | ||
71 | static inline int gro_cells_init(struct gro_cells *gcells, struct net_device *dev) | |
72 | { | |
73 | int i; | |
74 | ||
75 | gcells->gro_cells_mask = roundup_pow_of_two(netif_get_num_default_rss_queues()) - 1; | |
a1b1add0 JP |
76 | gcells->cells = kcalloc(gcells->gro_cells_mask + 1, |
77 | sizeof(struct gro_cell), | |
c9e6bc64 ED |
78 | GFP_KERNEL); |
79 | if (!gcells->cells) | |
80 | return -ENOMEM; | |
81 | ||
82 | for (i = 0; i <= gcells->gro_cells_mask; i++) { | |
83 | struct gro_cell *cell = gcells->cells + i; | |
84 | ||
85 | skb_queue_head_init(&cell->napi_skbs); | |
86 | netif_napi_add(dev, &cell->napi, gro_cell_poll, 64); | |
87 | napi_enable(&cell->napi); | |
88 | } | |
89 | return 0; | |
90 | } | |
91 | ||
92 | static inline void gro_cells_destroy(struct gro_cells *gcells) | |
93 | { | |
94 | struct gro_cell *cell = gcells->cells; | |
95 | int i; | |
96 | ||
97 | if (!cell) | |
98 | return; | |
99 | for (i = 0; i <= gcells->gro_cells_mask; i++,cell++) { | |
100 | netif_napi_del(&cell->napi); | |
101 | skb_queue_purge(&cell->napi_skbs); | |
102 | } | |
103 | kfree(gcells->cells); | |
104 | gcells->cells = NULL; | |
105 | } | |
106 | ||
107 | #endif |