Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-or-later
2 : /*
3 : * IPV4 GSO/GRO offload support
4 : * Linux INET implementation
5 : *
6 : * GRE GSO support
7 : */
8 :
9 : #include <linux/skbuff.h>
10 : #include <linux/init.h>
11 : #include <net/protocol.h>
12 : #include <net/gre.h>
13 :
14 0 : static struct sk_buff *gre_gso_segment(struct sk_buff *skb,
15 : netdev_features_t features)
16 : {
17 0 : int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
18 0 : bool need_csum, offload_csum, gso_partial, need_ipsec;
19 0 : struct sk_buff *segs = ERR_PTR(-EINVAL);
20 0 : u16 mac_offset = skb->mac_header;
21 0 : __be16 protocol = skb->protocol;
22 0 : u16 mac_len = skb->mac_len;
23 0 : int gre_offset, outer_hlen;
24 :
25 0 : if (!skb->encapsulation)
26 0 : goto out;
27 :
28 0 : if (unlikely(tnl_hlen < sizeof(struct gre_base_hdr)))
29 0 : goto out;
30 :
31 0 : if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
32 0 : goto out;
33 :
34 : /* setup inner skb. */
35 0 : skb->encapsulation = 0;
36 0 : SKB_GSO_CB(skb)->encap_level = 0;
37 0 : __skb_pull(skb, tnl_hlen);
38 0 : skb_reset_mac_header(skb);
39 0 : skb_set_network_header(skb, skb_inner_network_offset(skb));
40 0 : skb->mac_len = skb_inner_network_offset(skb);
41 0 : skb->protocol = skb->inner_protocol;
42 :
43 0 : need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM);
44 0 : skb->encap_hdr_csum = need_csum;
45 :
46 0 : features &= skb->dev->hw_enc_features;
47 0 : if (need_csum)
48 0 : features &= ~NETIF_F_SCTP_CRC;
49 :
50 0 : need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));
51 : /* Try to offload checksum if possible */
52 0 : offload_csum = !!(need_csum && !need_ipsec &&
53 0 : (skb->dev->features & NETIF_F_HW_CSUM));
54 :
55 : /* segment inner packet. */
56 0 : segs = skb_mac_gso_segment(skb, features);
57 0 : if (IS_ERR_OR_NULL(segs)) {
58 0 : skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
59 : mac_len);
60 0 : goto out;
61 : }
62 :
63 0 : gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
64 :
65 0 : outer_hlen = skb_tnl_header_len(skb);
66 0 : gre_offset = outer_hlen - tnl_hlen;
67 0 : skb = segs;
68 0 : do {
69 0 : struct gre_base_hdr *greh;
70 0 : __sum16 *pcsum;
71 :
72 : /* Set up inner headers if we are offloading inner checksum */
73 0 : if (skb->ip_summed == CHECKSUM_PARTIAL) {
74 0 : skb_reset_inner_headers(skb);
75 0 : skb->encapsulation = 1;
76 : }
77 :
78 0 : skb->mac_len = mac_len;
79 0 : skb->protocol = protocol;
80 :
81 0 : __skb_push(skb, outer_hlen);
82 0 : skb_reset_mac_header(skb);
83 0 : skb_set_network_header(skb, mac_len);
84 0 : skb_set_transport_header(skb, gre_offset);
85 :
86 0 : if (!need_csum)
87 0 : continue;
88 :
89 0 : greh = (struct gre_base_hdr *)skb_transport_header(skb);
90 0 : pcsum = (__sum16 *)(greh + 1);
91 :
92 0 : if (gso_partial && skb_is_gso(skb)) {
93 0 : unsigned int partial_adj;
94 :
95 : /* Adjust checksum to account for the fact that
96 : * the partial checksum is based on actual size
97 : * whereas headers should be based on MSS size.
98 : */
99 0 : partial_adj = skb->len + skb_headroom(skb) -
100 0 : SKB_GSO_CB(skb)->data_offset -
101 0 : skb_shinfo(skb)->gso_size;
102 0 : *pcsum = ~csum_fold((__force __wsum)htonl(partial_adj));
103 : } else {
104 0 : *pcsum = 0;
105 : }
106 :
107 0 : *(pcsum + 1) = 0;
108 0 : if (skb->encapsulation || !offload_csum) {
109 0 : *pcsum = gso_make_checksum(skb, 0);
110 : } else {
111 0 : skb->ip_summed = CHECKSUM_PARTIAL;
112 0 : skb->csum_start = skb_transport_header(skb) - skb->head;
113 0 : skb->csum_offset = sizeof(*greh);
114 : }
115 0 : } while ((skb = skb->next));
116 0 : out:
117 0 : return segs;
118 : }
119 :
120 0 : static struct sk_buff *gre_gro_receive(struct list_head *head,
121 : struct sk_buff *skb)
122 : {
123 0 : struct sk_buff *pp = NULL;
124 0 : struct sk_buff *p;
125 0 : const struct gre_base_hdr *greh;
126 0 : unsigned int hlen, grehlen;
127 0 : unsigned int off;
128 0 : int flush = 1;
129 0 : struct packet_offload *ptype;
130 0 : __be16 type;
131 :
132 0 : if (NAPI_GRO_CB(skb)->encap_mark)
133 0 : goto out;
134 :
135 0 : NAPI_GRO_CB(skb)->encap_mark = 1;
136 :
137 0 : off = skb_gro_offset(skb);
138 0 : hlen = off + sizeof(*greh);
139 0 : greh = skb_gro_header_fast(skb, off);
140 0 : if (skb_gro_header_hard(skb, hlen)) {
141 0 : greh = skb_gro_header_slow(skb, hlen, off);
142 0 : if (unlikely(!greh))
143 0 : goto out;
144 : }
145 :
146 : /* Only support version 0 and K (key), C (csum) flags. Note that
147 : * although the support for the S (seq#) flag can be added easily
148 : * for GRO, this is problematic for GSO hence can not be enabled
149 : * here because a GRO pkt may end up in the forwarding path, thus
150 : * requiring GSO support to break it up correctly.
151 : */
152 0 : if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0)
153 0 : goto out;
154 :
155 : /* We can only support GRE_CSUM if we can track the location of
156 : * the GRE header. In the case of FOU/GUE we cannot because the
157 : * outer UDP header displaces the GRE header leaving us in a state
158 : * of limbo.
159 : */
160 0 : if ((greh->flags & GRE_CSUM) && NAPI_GRO_CB(skb)->is_fou)
161 0 : goto out;
162 :
163 0 : type = greh->protocol;
164 :
165 0 : rcu_read_lock();
166 0 : ptype = gro_find_receive_by_type(type);
167 0 : if (!ptype)
168 0 : goto out_unlock;
169 :
170 0 : grehlen = GRE_HEADER_SECTION;
171 :
172 0 : if (greh->flags & GRE_KEY)
173 0 : grehlen += GRE_HEADER_SECTION;
174 :
175 0 : if (greh->flags & GRE_CSUM)
176 0 : grehlen += GRE_HEADER_SECTION;
177 :
178 0 : hlen = off + grehlen;
179 0 : if (skb_gro_header_hard(skb, hlen)) {
180 0 : greh = skb_gro_header_slow(skb, hlen, off);
181 0 : if (unlikely(!greh))
182 0 : goto out_unlock;
183 : }
184 :
185 : /* Don't bother verifying checksum if we're going to flush anyway. */
186 0 : if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) {
187 0 : if (skb_gro_checksum_simple_validate(skb))
188 0 : goto out_unlock;
189 :
190 0 : skb_gro_checksum_try_convert(skb, IPPROTO_GRE,
191 : null_compute_pseudo);
192 : }
193 :
194 0 : list_for_each_entry(p, head, list) {
195 0 : const struct gre_base_hdr *greh2;
196 :
197 0 : if (!NAPI_GRO_CB(p)->same_flow)
198 0 : continue;
199 :
200 : /* The following checks are needed to ensure only pkts
201 : * from the same tunnel are considered for aggregation.
202 : * The criteria for "the same tunnel" includes:
203 : * 1) same version (we only support version 0 here)
204 : * 2) same protocol (we only support ETH_P_IP for now)
205 : * 3) same set of flags
206 : * 4) same key if the key field is present.
207 : */
208 0 : greh2 = (struct gre_base_hdr *)(p->data + off);
209 :
210 0 : if (greh2->flags != greh->flags ||
211 0 : greh2->protocol != greh->protocol) {
212 0 : NAPI_GRO_CB(p)->same_flow = 0;
213 0 : continue;
214 : }
215 0 : if (greh->flags & GRE_KEY) {
216 : /* compare keys */
217 0 : if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) {
218 0 : NAPI_GRO_CB(p)->same_flow = 0;
219 0 : continue;
220 : }
221 : }
222 : }
223 :
224 0 : skb_gro_pull(skb, grehlen);
225 :
226 : /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
227 0 : skb_gro_postpull_rcsum(skb, greh, grehlen);
228 :
229 0 : pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
230 0 : flush = 0;
231 :
232 0 : out_unlock:
233 0 : rcu_read_unlock();
234 0 : out:
235 0 : skb_gro_flush_final(skb, pp, flush);
236 :
237 0 : return pp;
238 : }
239 :
240 0 : static int gre_gro_complete(struct sk_buff *skb, int nhoff)
241 : {
242 0 : struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff);
243 0 : struct packet_offload *ptype;
244 0 : unsigned int grehlen = sizeof(*greh);
245 0 : int err = -ENOENT;
246 0 : __be16 type;
247 :
248 0 : skb->encapsulation = 1;
249 0 : skb_shinfo(skb)->gso_type = SKB_GSO_GRE;
250 :
251 0 : type = greh->protocol;
252 0 : if (greh->flags & GRE_KEY)
253 0 : grehlen += GRE_HEADER_SECTION;
254 :
255 0 : if (greh->flags & GRE_CSUM)
256 0 : grehlen += GRE_HEADER_SECTION;
257 :
258 0 : rcu_read_lock();
259 0 : ptype = gro_find_complete_by_type(type);
260 0 : if (ptype)
261 0 : err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);
262 :
263 0 : rcu_read_unlock();
264 :
265 0 : skb_set_inner_mac_header(skb, nhoff + grehlen);
266 :
267 0 : return err;
268 : }
269 :
270 : static const struct net_offload gre_offload = {
271 : .callbacks = {
272 : .gso_segment = gre_gso_segment,
273 : .gro_receive = gre_gro_receive,
274 : .gro_complete = gre_gro_complete,
275 : },
276 : };
277 :
278 1 : static int __init gre_offload_init(void)
279 : {
280 1 : int err;
281 :
282 1 : err = inet_add_offload(&gre_offload, IPPROTO_GRE);
283 : #if IS_ENABLED(CONFIG_IPV6)
284 : if (err)
285 : return err;
286 :
287 : err = inet6_add_offload(&gre_offload, IPPROTO_GRE);
288 : if (err)
289 : inet_del_offload(&gre_offload, IPPROTO_GRE);
290 : #endif
291 :
292 1 : return err;
293 : }
294 : device_initcall(gre_offload_init);
|
