Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : /*
3 : * net/dst.h Protocol independent destination cache definitions.
4 : *
5 : * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6 : *
7 : */
8 :
9 : #ifndef _NET_DST_H
10 : #define _NET_DST_H
11 :
12 : #include <net/dst_ops.h>
13 : #include <linux/netdevice.h>
14 : #include <linux/rtnetlink.h>
15 : #include <linux/rcupdate.h>
16 : #include <linux/bug.h>
17 : #include <linux/jiffies.h>
18 : #include <linux/refcount.h>
19 : #include <net/neighbour.h>
20 : #include <asm/processor.h>
21 : #include <linux/indirect_call_wrapper.h>
22 :
23 : struct sk_buff;
24 :
25 : struct dst_entry {
26 : struct net_device *dev;
27 : struct dst_ops *ops;
28 : unsigned long _metrics;
29 : unsigned long expires;
30 : #ifdef CONFIG_XFRM
31 : struct xfrm_state *xfrm;
32 : #else
33 : void *__pad1;
34 : #endif
35 : int (*input)(struct sk_buff *);
36 : int (*output)(struct net *net, struct sock *sk, struct sk_buff *skb);
37 :
38 : unsigned short flags;
39 : #define DST_NOXFRM 0x0002
40 : #define DST_NOPOLICY 0x0004
41 : #define DST_NOCOUNT 0x0008
42 : #define DST_FAKE_RTABLE 0x0010
43 : #define DST_XFRM_TUNNEL 0x0020
44 : #define DST_XFRM_QUEUE 0x0040
45 : #define DST_METADATA 0x0080
46 :
47 : /* A non-zero value of dst->obsolete forces by-hand validation
48 : * of the route entry. Positive values are set by the generic
49 : * dst layer to indicate that the entry has been forcefully
50 : * destroyed.
51 : *
52 : * Negative values are used by the implementation layer code to
53 : * force invocation of the dst_ops->check() method.
54 : */
55 : short obsolete;
56 : #define DST_OBSOLETE_NONE 0
57 : #define DST_OBSOLETE_DEAD 2
58 : #define DST_OBSOLETE_FORCE_CHK -1
59 : #define DST_OBSOLETE_KILL -2
60 : unsigned short header_len; /* more space at head required */
61 : unsigned short trailer_len; /* space to reserve at tail */
62 :
63 : /*
64 : * __refcnt wants to be on a different cache line from
65 : * input/output/ops or performance tanks badly
66 : */
67 : #ifdef CONFIG_64BIT
68 : atomic_t __refcnt; /* 64-bit offset 64 */
69 : #endif
70 : int __use;
71 : unsigned long lastuse;
72 : struct lwtunnel_state *lwtstate;
73 : struct rcu_head rcu_head;
74 : short error;
75 : short __pad;
76 : __u32 tclassid;
77 : #ifndef CONFIG_64BIT
78 : atomic_t __refcnt; /* 32-bit offset 64 */
79 : #endif
80 : };
81 :
82 : struct dst_metrics {
83 : u32 metrics[RTAX_MAX];
84 : refcount_t refcnt;
85 : } __aligned(4); /* Low pointer bits contain DST_METRICS_FLAGS */
86 : extern const struct dst_metrics dst_default_metrics;
87 :
88 : u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old);
89 :
90 : #define DST_METRICS_READ_ONLY 0x1UL
91 : #define DST_METRICS_REFCOUNTED 0x2UL
92 : #define DST_METRICS_FLAGS 0x3UL
93 : #define __DST_METRICS_PTR(Y) \
94 : ((u32 *)((Y) & ~DST_METRICS_FLAGS))
95 : #define DST_METRICS_PTR(X) __DST_METRICS_PTR((X)->_metrics)
96 :
97 : static inline bool dst_metrics_read_only(const struct dst_entry *dst)
98 : {
99 : return dst->_metrics & DST_METRICS_READ_ONLY;
100 : }
101 :
102 : void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old);
103 :
104 : static inline void dst_destroy_metrics_generic(struct dst_entry *dst)
105 : {
106 : unsigned long val = dst->_metrics;
107 : if (!(val & DST_METRICS_READ_ONLY))
108 : __dst_destroy_metrics_generic(dst, val);
109 : }
110 :
111 : static inline u32 *dst_metrics_write_ptr(struct dst_entry *dst)
112 : {
113 : unsigned long p = dst->_metrics;
114 :
115 : BUG_ON(!p);
116 :
117 : if (p & DST_METRICS_READ_ONLY)
118 : return dst->ops->cow_metrics(dst, p);
119 : return __DST_METRICS_PTR(p);
120 : }
121 :
122 : /* This may only be invoked before the entry has reached global
123 : * visibility.
124 : */
125 23 : static inline void dst_init_metrics(struct dst_entry *dst,
126 : const u32 *src_metrics,
127 : bool read_only)
128 : {
129 23 : dst->_metrics = ((unsigned long) src_metrics) |
130 : (read_only ? DST_METRICS_READ_ONLY : 0);
131 : }
132 :
133 : static inline void dst_copy_metrics(struct dst_entry *dest, const struct dst_entry *src)
134 : {
135 : u32 *dst_metrics = dst_metrics_write_ptr(dest);
136 :
137 : if (dst_metrics) {
138 : u32 *src_metrics = DST_METRICS_PTR(src);
139 :
140 : memcpy(dst_metrics, src_metrics, RTAX_MAX * sizeof(u32));
141 : }
142 : }
143 :
144 0 : static inline u32 *dst_metrics_ptr(struct dst_entry *dst)
145 : {
146 0 : return DST_METRICS_PTR(dst);
147 : }
148 :
149 : static inline u32
150 3647 : dst_metric_raw(const struct dst_entry *dst, const int metric)
151 : {
152 3647 : u32 *p = DST_METRICS_PTR(dst);
153 :
154 1773 : return p[metric-1];
155 : }
156 :
157 : static inline u32
158 2210 : dst_metric(const struct dst_entry *dst, const int metric)
159 : {
160 4420 : WARN_ON_ONCE(metric == RTAX_HOPLIMIT ||
161 : metric == RTAX_ADVMSS ||
162 : metric == RTAX_MTU);
163 2210 : return dst_metric_raw(dst, metric);
164 : }
165 :
166 : static inline u32
167 12 : dst_metric_advmss(const struct dst_entry *dst)
168 : {
169 12 : u32 advmss = dst_metric_raw(dst, RTAX_ADVMSS);
170 :
171 12 : if (!advmss)
172 12 : advmss = dst->ops->default_advmss(dst);
173 :
174 12 : return advmss;
175 : }
176 :
177 : static inline void dst_metric_set(struct dst_entry *dst, int metric, u32 val)
178 : {
179 : u32 *p = dst_metrics_write_ptr(dst);
180 :
181 : if (p)
182 : p[metric-1] = val;
183 : }
184 :
185 : /* Kernel-internal feature bits that are unallocated in user space. */
186 : #define DST_FEATURE_ECN_CA (1U << 31)
187 :
188 : #define DST_FEATURE_MASK (DST_FEATURE_ECN_CA)
189 : #define DST_FEATURE_ECN_MASK (DST_FEATURE_ECN_CA | RTAX_FEATURE_ECN)
190 :
191 : static inline u32
192 0 : dst_feature(const struct dst_entry *dst, u32 feature)
193 : {
194 0 : return dst_metric(dst, RTAX_FEATURES) & feature;
195 : }
196 :
197 : INDIRECT_CALLABLE_DECLARE(unsigned int ip6_mtu(const struct dst_entry *));
198 : INDIRECT_CALLABLE_DECLARE(unsigned int ipv4_mtu(const struct dst_entry *));
199 968 : static inline u32 dst_mtu(const struct dst_entry *dst)
200 : {
201 968 : return INDIRECT_CALL_INET(dst->ops->mtu, ip6_mtu, ipv4_mtu, dst);
202 : }
203 :
204 : /* RTT metrics are stored in milliseconds for user ABI, but used as jiffies */
205 0 : static inline unsigned long dst_metric_rtt(const struct dst_entry *dst, int metric)
206 : {
207 0 : return msecs_to_jiffies(dst_metric(dst, metric));
208 : }
209 :
210 : static inline u32
211 0 : dst_allfrag(const struct dst_entry *dst)
212 : {
213 0 : int ret = dst_feature(dst, RTAX_FEATURE_ALLFRAG);
214 0 : return ret;
215 : }
216 :
217 : static inline int
218 2132 : dst_metric_locked(const struct dst_entry *dst, int metric)
219 : {
220 2128 : return dst_metric(dst, RTAX_LOCK) & (1 << metric);
221 : }
222 :
223 10 : static inline void dst_hold(struct dst_entry *dst)
224 : {
225 : /*
226 : * If your kernel compilation stops here, please check
227 : * the placement of __refcnt in struct dst_entry
228 : */
229 10 : BUILD_BUG_ON(offsetof(struct dst_entry, __refcnt) & 63);
230 20 : WARN_ON(atomic_inc_not_zero(&dst->__refcnt) == 0);
231 10 : }
232 :
233 : static inline void dst_use_noref(struct dst_entry *dst, unsigned long time)
234 : {
235 : if (unlikely(time != dst->lastuse)) {
236 : dst->__use++;
237 : dst->lastuse = time;
238 : }
239 : }
240 :
241 : static inline void dst_hold_and_use(struct dst_entry *dst, unsigned long time)
242 : {
243 : dst_hold(dst);
244 : dst_use_noref(dst, time);
245 : }
246 :
247 1188 : static inline struct dst_entry *dst_clone(struct dst_entry *dst)
248 : {
249 1188 : if (dst)
250 2 : dst_hold(dst);
251 4 : return dst;
252 : }
253 :
254 : void dst_release(struct dst_entry *dst);
255 :
256 : void dst_release_immediate(struct dst_entry *dst);
257 :
258 898 : static inline void refdst_drop(unsigned long refdst)
259 : {
260 898 : if (!(refdst & SKB_DST_NOREF))
261 22 : dst_release((struct dst_entry *)(refdst & SKB_DST_PTRMASK));
262 898 : }
263 :
264 : /**
265 : * skb_dst_drop - drops skb dst
266 : * @skb: buffer
267 : *
268 : * Drops dst reference count if a reference was taken.
269 : */
270 6693 : static inline void skb_dst_drop(struct sk_buff *skb)
271 : {
272 6693 : if (skb->_skb_refdst) {
273 898 : refdst_drop(skb->_skb_refdst);
274 898 : skb->_skb_refdst = 0UL;
275 : }
276 0 : }
277 :
278 1233 : static inline void __skb_dst_copy(struct sk_buff *nskb, unsigned long refdst)
279 : {
280 1233 : nskb->_skb_refdst = refdst;
281 1233 : if (!(nskb->_skb_refdst & SKB_DST_NOREF))
282 1184 : dst_clone(skb_dst(nskb));
283 1233 : }
284 :
285 1233 : static inline void skb_dst_copy(struct sk_buff *nskb, const struct sk_buff *oskb)
286 : {
287 1233 : __skb_dst_copy(nskb, oskb->_skb_refdst);
288 : }
289 :
290 : /**
291 : * dst_hold_safe - Take a reference on a dst if possible
292 : * @dst: pointer to dst entry
293 : *
294 : * This helper returns false if it could not safely
295 : * take a reference on a dst.
296 : */
297 29 : static inline bool dst_hold_safe(struct dst_entry *dst)
298 : {
299 29 : return atomic_inc_not_zero(&dst->__refcnt);
300 : }
301 :
302 : /**
303 : * skb_dst_force - makes sure skb dst is refcounted
304 : * @skb: buffer
305 : *
306 : * If dst is not yet refcounted and not destroyed, grab a ref on it.
307 : * Returns true if dst is refcounted.
308 : */
309 144 : static inline bool skb_dst_force(struct sk_buff *skb)
310 : {
311 144 : if (skb_dst_is_noref(skb)) {
312 0 : struct dst_entry *dst = skb_dst(skb);
313 :
314 0 : WARN_ON(!rcu_read_lock_held());
315 0 : if (!dst_hold_safe(dst))
316 0 : dst = NULL;
317 :
318 0 : skb->_skb_refdst = (unsigned long)dst;
319 : }
320 :
321 144 : return skb->_skb_refdst != 0UL;
322 : }
323 :
324 :
325 : /**
326 : * __skb_tunnel_rx - prepare skb for rx reinsert
327 : * @skb: buffer
328 : * @dev: tunnel device
329 : * @net: netns for packet i/o
330 : *
331 : * After decapsulation, packet is going to re-enter (netif_rx()) our stack,
332 : * so make some cleanups. (no accounting done)
333 : */
334 : static inline void __skb_tunnel_rx(struct sk_buff *skb, struct net_device *dev,
335 : struct net *net)
336 : {
337 : skb->dev = dev;
338 :
339 : /*
340 : * Clear hash so that we can recalulate the hash for the
341 : * encapsulated packet, unless we have already determine the hash
342 : * over the L4 4-tuple.
343 : */
344 : skb_clear_hash_if_not_l4(skb);
345 : skb_set_queue_mapping(skb, 0);
346 : skb_scrub_packet(skb, !net_eq(net, dev_net(dev)));
347 : }
348 :
349 : /**
350 : * skb_tunnel_rx - prepare skb for rx reinsert
351 : * @skb: buffer
352 : * @dev: tunnel device
353 : * @net: netns for packet i/o
354 : *
355 : * After decapsulation, packet is going to re-enter (netif_rx()) our stack,
356 : * so make some cleanups, and perform accounting.
357 : * Note: this accounting is not SMP safe.
358 : */
359 : static inline void skb_tunnel_rx(struct sk_buff *skb, struct net_device *dev,
360 : struct net *net)
361 : {
362 : /* TODO : stats should be SMP safe */
363 : dev->stats.rx_packets++;
364 : dev->stats.rx_bytes += skb->len;
365 : __skb_tunnel_rx(skb, dev, net);
366 : }
367 :
368 0 : static inline u32 dst_tclassid(const struct sk_buff *skb)
369 : {
370 : #ifdef CONFIG_IP_ROUTE_CLASSID
371 : const struct dst_entry *dst;
372 :
373 : dst = skb_dst(skb);
374 : if (dst)
375 : return dst->tclassid;
376 : #endif
377 0 : return 0;
378 : }
379 :
380 : int dst_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
381 0 : static inline int dst_discard(struct sk_buff *skb)
382 : {
383 0 : return dst_discard_out(&init_net, skb->sk, skb);
384 : }
385 : void *dst_alloc(struct dst_ops *ops, struct net_device *dev, int initial_ref,
386 : int initial_obsolete, unsigned short flags);
387 : void dst_init(struct dst_entry *dst, struct dst_ops *ops,
388 : struct net_device *dev, int initial_ref, int initial_obsolete,
389 : unsigned short flags);
390 : struct dst_entry *dst_destroy(struct dst_entry *dst);
391 : void dst_dev_put(struct dst_entry *dst);
392 :
393 : static inline void dst_confirm(struct dst_entry *dst)
394 : {
395 : }
396 :
397 : static inline struct neighbour *dst_neigh_lookup(const struct dst_entry *dst, const void *daddr)
398 : {
399 : struct neighbour *n = dst->ops->neigh_lookup(dst, NULL, daddr);
400 : return IS_ERR(n) ? NULL : n;
401 : }
402 :
403 2 : static inline struct neighbour *dst_neigh_lookup_skb(const struct dst_entry *dst,
404 : struct sk_buff *skb)
405 : {
406 2 : struct neighbour *n;
407 :
408 2 : if (WARN_ON_ONCE(!dst->ops->neigh_lookup))
409 : return NULL;
410 :
411 2 : n = dst->ops->neigh_lookup(dst, skb, NULL);
412 :
413 2 : return IS_ERR(n) ? NULL : n;
414 : }
415 :
416 0 : static inline void dst_confirm_neigh(const struct dst_entry *dst,
417 : const void *daddr)
418 : {
419 0 : if (dst->ops->confirm_neigh)
420 0 : dst->ops->confirm_neigh(dst, daddr);
421 : }
422 :
423 0 : static inline void dst_link_failure(struct sk_buff *skb)
424 : {
425 0 : struct dst_entry *dst = skb_dst(skb);
426 0 : if (dst && dst->ops && dst->ops->link_failure)
427 0 : dst->ops->link_failure(skb);
428 0 : }
429 :
430 0 : static inline void dst_set_expires(struct dst_entry *dst, int timeout)
431 : {
432 0 : unsigned long expires = jiffies + timeout;
433 :
434 0 : if (expires == 0)
435 : expires = 1;
436 :
437 0 : if (dst->expires == 0 || time_before(expires, dst->expires))
438 0 : dst->expires = expires;
439 : }
440 :
441 : INDIRECT_CALLABLE_DECLARE(int ip6_output(struct net *, struct sock *,
442 : struct sk_buff *));
443 : INDIRECT_CALLABLE_DECLARE(int ip_output(struct net *, struct sock *,
444 : struct sk_buff *));
445 : /* Output packet to network from transport. */
446 444 : static inline int dst_output(struct net *net, struct sock *sk, struct sk_buff *skb)
447 : {
448 444 : return INDIRECT_CALL_INET(skb_dst(skb)->output,
449 : ip6_output, ip_output,
450 : net, sk, skb);
451 : }
452 :
453 : INDIRECT_CALLABLE_DECLARE(int ip6_input(struct sk_buff *));
454 : INDIRECT_CALLABLE_DECLARE(int ip_local_deliver(struct sk_buff *));
455 : /* Input packet from network to transport. */
456 454 : static inline int dst_input(struct sk_buff *skb)
457 : {
458 454 : return INDIRECT_CALL_INET(skb_dst(skb)->input,
459 : ip6_input, ip_local_deliver, skb);
460 : }
461 :
462 : INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *,
463 : u32));
464 : INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
465 : u32));
466 381 : static inline struct dst_entry *dst_check(struct dst_entry *dst, u32 cookie)
467 : {
468 381 : if (dst->obsolete)
469 381 : dst = INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check,
470 : ipv4_dst_check, dst, cookie);
471 0 : return dst;
472 : }
473 :
474 : /* Flags for xfrm_lookup flags argument. */
475 : enum {
476 : XFRM_LOOKUP_ICMP = 1 << 0,
477 : XFRM_LOOKUP_QUEUE = 1 << 1,
478 : XFRM_LOOKUP_KEEP_DST_REF = 1 << 2,
479 : };
480 :
481 : struct flowi;
482 : #ifndef CONFIG_XFRM
483 0 : static inline struct dst_entry *xfrm_lookup(struct net *net,
484 : struct dst_entry *dst_orig,
485 : const struct flowi *fl,
486 : const struct sock *sk,
487 : int flags)
488 : {
489 0 : return dst_orig;
490 : }
491 :
492 : static inline struct dst_entry *
493 : xfrm_lookup_with_ifid(struct net *net, struct dst_entry *dst_orig,
494 : const struct flowi *fl, const struct sock *sk,
495 : int flags, u32 if_id)
496 : {
497 : return dst_orig;
498 : }
499 :
500 25 : static inline struct dst_entry *xfrm_lookup_route(struct net *net,
501 : struct dst_entry *dst_orig,
502 : const struct flowi *fl,
503 : const struct sock *sk,
504 : int flags)
505 : {
506 25 : return dst_orig;
507 : }
508 :
509 0 : static inline struct xfrm_state *dst_xfrm(const struct dst_entry *dst)
510 : {
511 0 : return NULL;
512 : }
513 :
514 : #else
515 : struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
516 : const struct flowi *fl, const struct sock *sk,
517 : int flags);
518 :
519 : struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
520 : struct dst_entry *dst_orig,
521 : const struct flowi *fl,
522 : const struct sock *sk, int flags,
523 : u32 if_id);
524 :
525 : struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
526 : const struct flowi *fl, const struct sock *sk,
527 : int flags);
528 :
529 : /* skb attached with this dst needs transformation if dst->xfrm is valid */
530 : static inline struct xfrm_state *dst_xfrm(const struct dst_entry *dst)
531 : {
532 : return dst->xfrm;
533 : }
534 : #endif
535 :
536 : static inline void skb_dst_update_pmtu(struct sk_buff *skb, u32 mtu)
537 : {
538 : struct dst_entry *dst = skb_dst(skb);
539 :
540 : if (dst && dst->ops->update_pmtu)
541 : dst->ops->update_pmtu(dst, NULL, skb, mtu, true);
542 : }
543 :
544 : /* update dst pmtu but not do neighbor confirm */
545 0 : static inline void skb_dst_update_pmtu_no_confirm(struct sk_buff *skb, u32 mtu)
546 : {
547 0 : struct dst_entry *dst = skb_dst(skb);
548 :
549 0 : if (dst && dst->ops->update_pmtu)
550 0 : dst->ops->update_pmtu(dst, NULL, skb, mtu, false);
551 0 : }
552 :
553 : #endif /* _NET_DST_H */
|
