LCOV - code coverage report
Current view: top level - net/ipv4 - tcp_output.c (source / functions) Hit Total Coverage
Test: landlock.info Lines: 676 1755 38.5 %
Date: 2021-04-22 12:43:58 Functions: 48 89 53.9 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0-only
       2             : /*
       3             :  * INET         An implementation of the TCP/IP protocol suite for the LINUX
       4             :  *              operating system.  INET is implemented using the  BSD Socket
       5             :  *              interface as the means of communication with the user level.
       6             :  *
       7             :  *              Implementation of the Transmission Control Protocol(TCP).
       8             :  *
       9             :  * Authors:     Ross Biro
      10             :  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
      11             :  *              Mark Evans, <evansmp@uhura.aston.ac.uk>
      12             :  *              Corey Minyard <wf-rch!minyard@relay.EU.net>
      13             :  *              Florian La Roche, <flla@stud.uni-sb.de>
      14             :  *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
      15             :  *              Linus Torvalds, <torvalds@cs.helsinki.fi>
      16             :  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
      17             :  *              Matthew Dillon, <dillon@apollo.west.oic.com>
      18             :  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
      19             :  *              Jorge Cwik, <jorge@laser.satlink.net>
      20             :  */
      21             : 
      22             : /*
      23             :  * Changes:     Pedro Roque     :       Retransmit queue handled by TCP.
      24             :  *                              :       Fragmentation on mtu decrease
      25             :  *                              :       Segment collapse on retransmit
      26             :  *                              :       AF independence
      27             :  *
      28             :  *              Linus Torvalds  :       send_delayed_ack
      29             :  *              David S. Miller :       Charge memory using the right skb
      30             :  *                                      during syn/ack processing.
      31             :  *              David S. Miller :       Output engine completely rewritten.
      32             :  *              Andrea Arcangeli:       SYNACK carry ts_recent in tsecr.
      33             :  *              Cacophonix Gaul :       draft-minshall-nagle-01
      34             :  *              J Hadi Salim    :       ECN support
      35             :  *
      36             :  */
      37             : 
      38             : #define pr_fmt(fmt) "TCP: " fmt
      39             : 
      40             : #include <net/tcp.h>
      41             : #include <net/mptcp.h>
      42             : 
      43             : #include <linux/compiler.h>
      44             : #include <linux/gfp.h>
      45             : #include <linux/module.h>
      46             : #include <linux/static_key.h>
      47             : 
      48             : #include <trace/events/tcp.h>
      49             : 
      50             : /* Refresh clocks of a TCP socket,
      51             :  * ensuring monotically increasing values.
      52             :  */
      53        1062 : void tcp_mstamp_refresh(struct tcp_sock *tp)
      54             : {
      55         640 :         u64 val = tcp_clock_ns();
      56             : 
      57        1062 :         tp->tcp_clock_cache = val;
      58         422 :         tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC);
      59         640 : }
      60             : 
      61             : static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
      62             :                            int push_one, gfp_t gfp);
      63             : 
      64             : /* Account for new data that has been sent to the network. */
      65         364 : static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb)
      66             : {
      67         364 :         struct inet_connection_sock *icsk = inet_csk(sk);
      68         364 :         struct tcp_sock *tp = tcp_sk(sk);
      69         364 :         unsigned int prior_packets = tp->packets_out;
      70             : 
      71         364 :         WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(skb)->end_seq);
      72             : 
      73         364 :         __skb_unlink(skb, &sk->sk_write_queue);
      74         364 :         tcp_rbtree_insert(&sk->tcp_rtx_queue, skb);
      75             : 
      76         364 :         if (tp->highest_sack == NULL)
      77         234 :                 tp->highest_sack = skb;
      78             : 
      79         364 :         tp->packets_out += tcp_skb_pcount(skb);
      80         364 :         if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)
      81         234 :                 tcp_rearm_rto(sk);
      82             : 
      83         364 :         NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
      84             :                       tcp_skb_pcount(skb));
      85         364 : }
      86             : 
      87             : /* SND.NXT, if window was not shrunk or the amount of shrunk was less than one
      88             :  * window scaling factor due to loss of precision.
      89             :  * If window has been shrunk, what should we make? It is not clear at all.
      90             :  * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
      91             :  * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
      92             :  * invalid. OK, let's make this for now:
      93             :  */
      94          62 : static inline __u32 tcp_acceptable_seq(const struct sock *sk)
      95             : {
      96          62 :         const struct tcp_sock *tp = tcp_sk(sk);
      97             : 
      98          62 :         if (!before(tcp_wnd_end(tp), tp->snd_nxt) ||
      99           0 :             (tp->rx_opt.wscale_ok &&
     100           0 :              ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale))))
     101             :                 return tp->snd_nxt;
     102             :         else
     103           0 :                 return tcp_wnd_end(tp);
     104             : }
     105             : 
     106             : /* Calculate mss to advertise in SYN segment.
     107             :  * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
     108             :  *
     109             :  * 1. It is independent of path mtu.
     110             :  * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
     111             :  * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
     112             :  *    attached devices, because some buggy hosts are confused by
     113             :  *    large MSS.
     114             :  * 4. We do not make 3, we advertise MSS, calculated from first
     115             :  *    hop device mtu, but allow to raise it to ip_rt_min_advmss.
     116             :  *    This may be overridden via information stored in routing table.
     117             :  * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
     118             :  *    probably even Jumbo".
     119             :  */
     120           0 : static __u16 tcp_advertise_mss(struct sock *sk)
     121             : {
     122           0 :         struct tcp_sock *tp = tcp_sk(sk);
     123           0 :         const struct dst_entry *dst = __sk_dst_get(sk);
     124           0 :         int mss = tp->advmss;
     125             : 
     126           0 :         if (dst) {
     127           0 :                 unsigned int metric = dst_metric_advmss(dst);
     128             : 
     129           0 :                 if (metric < mss) {
     130           0 :                         mss = metric;
     131           0 :                         tp->advmss = mss;
     132             :                 }
     133             :         }
     134             : 
     135           0 :         return (__u16)mss;
     136             : }
     137             : 
     138             : /* RFC2861. Reset CWND after idle period longer RTO to "restart window".
     139             :  * This is the first part of cwnd validation mechanism.
     140             :  */
     141           4 : void tcp_cwnd_restart(struct sock *sk, s32 delta)
     142             : {
     143           4 :         struct tcp_sock *tp = tcp_sk(sk);
     144           4 :         u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
     145           4 :         u32 cwnd = tp->snd_cwnd;
     146             : 
     147           4 :         tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
     148             : 
     149           4 :         tp->snd_ssthresh = tcp_current_ssthresh(sk);
     150           4 :         restart_cwnd = min(restart_cwnd, cwnd);
     151             : 
     152           4 :         while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
     153           0 :                 cwnd >>= 1;
     154           4 :         tp->snd_cwnd = max(cwnd, restart_cwnd);
     155           4 :         tp->snd_cwnd_stamp = tcp_jiffies32;
     156           4 :         tp->snd_cwnd_used = 0;
     157           4 : }
     158             : 
     159             : /* Congestion state accounting after a packet has been sent. */
     160         361 : static void tcp_event_data_sent(struct tcp_sock *tp,
     161             :                                 struct sock *sk)
     162             : {
     163         361 :         struct inet_connection_sock *icsk = inet_csk(sk);
     164         361 :         const u32 now = tcp_jiffies32;
     165             : 
     166         361 :         if (tcp_packets_in_flight(tp) == 0)
     167         231 :                 tcp_ca_event(sk, CA_EVENT_TX_START);
     168             : 
     169             :         /* If this is the first data packet sent in response to the
     170             :          * previous received data,
     171             :          * and it is a reply for ato after last received packet,
     172             :          * increase pingpong count.
     173             :          */
     174         361 :         if (before(tp->lsndtime, icsk->icsk_ack.lrcvtime) &&
     175          16 :             (u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
     176          15 :                 inet_csk_inc_pingpong_cnt(sk);
     177             : 
     178         361 :         tp->lsndtime = now;
     179         361 : }
     180             : 
     181             : /* Account for an ACK we sent. */
     182         426 : static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts,
     183             :                                       u32 rcv_nxt)
     184             : {
     185         426 :         struct tcp_sock *tp = tcp_sk(sk);
     186             : 
     187         426 :         if (unlikely(tp->compressed_ack)) {
     188           0 :                 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
     189             :                               tp->compressed_ack);
     190           0 :                 tp->compressed_ack = 0;
     191           0 :                 if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)
     192           0 :                         __sock_put(sk);
     193             :         }
     194             : 
     195         426 :         if (unlikely(rcv_nxt != tp->rcv_nxt))
     196             :                 return;  /* Special ACK sent by DCTCP to reflect ECN */
     197         426 :         tcp_dec_quickack_mode(sk, pkts);
     198         426 :         inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
     199             : }
     200             : 
     201             : /* Determine a window scaling and initial window to offer.
     202             :  * Based on the assumption that the given amount of space
     203             :  * will be offered. Store the results in the tp structure.
     204             :  * NOTE: for smooth operation initial space offering should
     205             :  * be a multiple of mss if possible. We assume here that mss >= 1.
     206             :  * This MUST be enforced by all callers.
     207             :  */
     208           4 : void tcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
     209             :                                __u32 *rcv_wnd, __u32 *window_clamp,
     210             :                                int wscale_ok, __u8 *rcv_wscale,
     211             :                                __u32 init_rcv_wnd)
     212             : {
     213           4 :         unsigned int space = (__space < 0 ? 0 : __space);
     214             : 
     215             :         /* If no clamp set the clamp to the max possible scaled window */
     216           4 :         if (*window_clamp == 0)
     217           4 :                 (*window_clamp) = (U16_MAX << TCP_MAX_WSCALE);
     218           4 :         space = min(*window_clamp, space);
     219             : 
     220             :         /* Quantize space offering to a multiple of mss if possible. */
     221           4 :         if (space > mss)
     222           4 :                 space = rounddown(space, mss);
     223             : 
     224             :         /* NOTE: offering an initial window larger than 32767
     225             :          * will break some buggy TCP stacks. If the admin tells us
     226             :          * it is likely we could be speaking with such a buggy stack
     227             :          * we will truncate our initial window offering to 32K-1
     228             :          * unless the remote has sent us a window scaling option,
     229             :          * which we interpret as a sign the remote TCP is not
     230             :          * misinterpreting the window field as a signed quantity.
     231             :          */
     232           4 :         if (sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows)
     233           0 :                 (*rcv_wnd) = min(space, MAX_TCP_WINDOW);
     234             :         else
     235           4 :                 (*rcv_wnd) = min_t(u32, space, U16_MAX);
     236             : 
     237           4 :         if (init_rcv_wnd)
     238           0 :                 *rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
     239             : 
     240           4 :         *rcv_wscale = 0;
     241           4 :         if (wscale_ok) {
     242             :                 /* Set window scaling on max possible window */
     243           0 :                 space = max_t(u32, space, sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
     244           0 :                 space = max_t(u32, space, sysctl_rmem_max);
     245           0 :                 space = min_t(u32, space, *window_clamp);
     246           0 :                 *rcv_wscale = clamp_t(int, ilog2(space) - 15,
     247             :                                       0, TCP_MAX_WSCALE);
     248             :         }
     249             :         /* Set the clamp no higher than max representable value */
     250           4 :         (*window_clamp) = min_t(__u32, U16_MAX << (*rcv_wscale), *window_clamp);
     251           4 : }
     252             : EXPORT_SYMBOL(tcp_select_initial_window);
     253             : 
     254             : /* Chose a new window to advertise, update state in tcp_sock for the
     255             :  * socket, and return result with RFC1323 scaling applied.  The return
     256             :  * value can be stuffed directly into th->window for an outgoing
     257             :  * frame.
     258             :  */
     259         426 : static u16 tcp_select_window(struct sock *sk)
     260             : {
     261         426 :         struct tcp_sock *tp = tcp_sk(sk);
     262         426 :         u32 old_win = tp->rcv_wnd;
     263         426 :         u32 cur_win = tcp_receive_window(tp);
     264         426 :         u32 new_win = __tcp_select_window(sk);
     265             : 
     266             :         /* Never shrink the offered window */
     267         426 :         if (new_win < cur_win) {
     268             :                 /* Danger Will Robinson!
     269             :                  * Don't update rcv_wup/rcv_wnd here or else
     270             :                  * we will not be able to advertise a zero
     271             :                  * window in time.  --DaveM
     272             :                  *
     273             :                  * Relax Will Robinson.
     274             :                  */
     275           9 :                 if (new_win == 0)
     276           9 :                         NET_INC_STATS(sock_net(sk),
     277             :                                       LINUX_MIB_TCPWANTZEROWINDOWADV);
     278           9 :                 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
     279             :         }
     280         426 :         tp->rcv_wnd = new_win;
     281         426 :         tp->rcv_wup = tp->rcv_nxt;
     282             : 
     283             :         /* Make sure we do not exceed the maximum possible
     284             :          * scaled window.
     285             :          */
     286         426 :         if (!tp->rx_opt.rcv_wscale &&
     287         426 :             sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows)
     288           0 :                 new_win = min(new_win, MAX_TCP_WINDOW);
     289             :         else
     290         426 :                 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
     291             : 
     292             :         /* RFC1323 scaling applied */
     293         426 :         new_win >>= tp->rx_opt.rcv_wscale;
     294             : 
     295             :         /* If we advertise zero window, disable fast path. */
     296         426 :         if (new_win == 0) {
     297           0 :                 tp->pred_flags = 0;
     298           0 :                 if (old_win)
     299           0 :                         NET_INC_STATS(sock_net(sk),
     300             :                                       LINUX_MIB_TCPTOZEROWINDOWADV);
     301         426 :         } else if (old_win == 0) {
     302         426 :                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFROMZEROWINDOWADV);
     303             :         }
     304             : 
     305         426 :         return new_win;
     306             : }
     307             : 
     308             : /* Packet ECN state for a SYN-ACK */
     309           0 : static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb)
     310             : {
     311           0 :         const struct tcp_sock *tp = tcp_sk(sk);
     312             : 
     313           0 :         TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
     314           0 :         if (!(tp->ecn_flags & TCP_ECN_OK))
     315           0 :                 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
     316           0 :         else if (tcp_ca_needs_ecn(sk) ||
     317           0 :                  tcp_bpf_ca_needs_ecn(sk))
     318           0 :                 INET_ECN_xmit(sk);
     319           0 : }
     320             : 
     321             : /* Packet ECN state for a SYN.  */
     322           0 : static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb)
     323             : {
     324           0 :         struct tcp_sock *tp = tcp_sk(sk);
     325           0 :         bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk);
     326           0 :         bool use_ecn = sock_net(sk)->ipv4.sysctl_tcp_ecn == 1 ||
     327           0 :                 tcp_ca_needs_ecn(sk) || bpf_needs_ecn;
     328             : 
     329           0 :         if (!use_ecn) {
     330           0 :                 const struct dst_entry *dst = __sk_dst_get(sk);
     331             : 
     332           0 :                 if (dst && dst_feature(dst, RTAX_FEATURE_ECN))
     333           0 :                         use_ecn = true;
     334             :         }
     335             : 
     336           0 :         tp->ecn_flags = 0;
     337             : 
     338           0 :         if (use_ecn) {
     339           0 :                 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
     340           0 :                 tp->ecn_flags = TCP_ECN_OK;
     341           0 :                 if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn)
     342           0 :                         INET_ECN_xmit(sk);
     343             :         }
     344           0 : }
     345             : 
     346           0 : static void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb)
     347             : {
     348           0 :         if (sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback)
     349             :                 /* tp->ecn_flags are cleared at a later point in time when
     350             :                  * SYN ACK is ultimatively being received.
     351             :                  */
     352           0 :                 TCP_SKB_CB(skb)->tcp_flags &= ~(TCPHDR_ECE | TCPHDR_CWR);
     353             : }
     354             : 
     355             : static void
     356           4 : tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th)
     357             : {
     358           4 :         if (inet_rsk(req)->ecn_ok)
     359           0 :                 th->ece = 1;
     360             : }
     361             : 
     362             : /* Set up ECN state for a packet on a ESTABLISHED socket that is about to
     363             :  * be sent.
     364             :  */
     365         426 : static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
     366             :                          struct tcphdr *th, int tcp_header_len)
     367             : {
     368         426 :         struct tcp_sock *tp = tcp_sk(sk);
     369             : 
     370         426 :         if (tp->ecn_flags & TCP_ECN_OK) {
     371             :                 /* Not-retransmitted data segment: set ECT and inject CWR. */
     372           0 :                 if (skb->len != tcp_header_len &&
     373           0 :                     !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
     374           0 :                         INET_ECN_xmit(sk);
     375           0 :                         if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
     376           0 :                                 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
     377           0 :                                 th->cwr = 1;
     378           0 :                                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
     379             :                         }
     380           0 :                 } else if (!tcp_ca_needs_ecn(sk)) {
     381             :                         /* ACK or retransmitted segment: clear ECT|CE */
     382           0 :                         INET_ECN_dontxmit(sk);
     383             :                 }
     384           0 :                 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
     385           0 :                         th->ece = 1;
     386             :         }
     387         426 : }
     388             : 
     389             : /* Constructs common control bits of non-data skb. If SYN/FIN is present,
     390             :  * auto increment end seqno.
     391             :  */
     392          65 : static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
     393             : {
     394          65 :         skb->ip_summed = CHECKSUM_PARTIAL;
     395             : 
     396          65 :         TCP_SKB_CB(skb)->tcp_flags = flags;
     397          65 :         TCP_SKB_CB(skb)->sacked = 0;
     398             : 
     399          65 :         tcp_skb_pcount_set(skb, 1);
     400             : 
     401          65 :         TCP_SKB_CB(skb)->seq = seq;
     402          65 :         if (flags & (TCPHDR_SYN | TCPHDR_FIN))
     403           3 :                 seq++;
     404          65 :         TCP_SKB_CB(skb)->end_seq = seq;
     405             : }
     406             : 
     407         792 : static inline bool tcp_urg_mode(const struct tcp_sock *tp)
     408             : {
     409         792 :         return tp->snd_una != tp->snd_up;
     410             : }
     411             : 
     412             : #define OPTION_SACK_ADVERTISE   (1 << 0)
     413             : #define OPTION_TS               (1 << 1)
     414             : #define OPTION_MD5              (1 << 2)
     415             : #define OPTION_WSCALE           (1 << 3)
     416             : #define OPTION_FAST_OPEN_COOKIE (1 << 8)
     417             : #define OPTION_SMC              (1 << 9)
     418             : #define OPTION_MPTCP            (1 << 10)
     419             : 
     420         430 : static void smc_options_write(__be32 *ptr, u16 *options)
     421             : {
     422             : #if IS_ENABLED(CONFIG_SMC)
     423             :         if (static_branch_unlikely(&tcp_have_smc)) {
     424             :                 if (unlikely(OPTION_SMC & *options)) {
     425             :                         *ptr++ = htonl((TCPOPT_NOP  << 24) |
     426             :                                        (TCPOPT_NOP  << 16) |
     427             :                                        (TCPOPT_EXP <<  8) |
     428             :                                        (TCPOLEN_EXP_SMC_BASE));
     429             :                         *ptr++ = htonl(TCPOPT_SMC_MAGIC);
     430             :                 }
     431             :         }
     432             : #endif
     433         430 : }
     434             : 
     435             : struct tcp_out_options {
     436             :         u16 options;            /* bit field of OPTION_* */
     437             :         u16 mss;                /* 0 to disable */
     438             :         u8 ws;                  /* window scale, 0 to disable */
     439             :         u8 num_sack_blocks;     /* number of SACK blocks to include */
     440             :         u8 hash_size;           /* bytes in hash_location */
     441             :         u8 bpf_opt_len;         /* length of BPF hdr option */
     442             :         __u8 *hash_location;    /* temporary pointer, overloaded */
     443             :         __u32 tsval, tsecr;     /* need to include OPTION_TS */
     444             :         struct tcp_fastopen_cookie *fastopen_cookie;    /* Fast open cookie */
     445             :         struct mptcp_out_options mptcp;
     446             : };
     447             : 
     448         430 : static void mptcp_options_write(__be32 *ptr, const struct tcp_sock *tp,
     449             :                                 struct tcp_out_options *opts)
     450             : {
     451             : #if IS_ENABLED(CONFIG_MPTCP)
     452             :         if (unlikely(OPTION_MPTCP & opts->options))
     453             :                 mptcp_write_options(ptr, tp, &opts->mptcp);
     454             : #endif
     455         430 : }
     456             : 
     457             : #ifdef CONFIG_CGROUP_BPF
     458             : static int bpf_skops_write_hdr_opt_arg0(struct sk_buff *skb,
     459             :                                         enum tcp_synack_type synack_type)
     460             : {
     461             :         if (unlikely(!skb))
     462             :                 return BPF_WRITE_HDR_TCP_CURRENT_MSS;
     463             : 
     464             :         if (unlikely(synack_type == TCP_SYNACK_COOKIE))
     465             :                 return BPF_WRITE_HDR_TCP_SYNACK_COOKIE;
     466             : 
     467             :         return 0;
     468             : }
     469             : 
     470             : /* req, syn_skb and synack_type are used when writing synack */
     471             : static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
     472             :                                   struct request_sock *req,
     473             :                                   struct sk_buff *syn_skb,
     474             :                                   enum tcp_synack_type synack_type,
     475             :                                   struct tcp_out_options *opts,
     476             :                                   unsigned int *remaining)
     477             : {
     478             :         struct bpf_sock_ops_kern sock_ops;
     479             :         int err;
     480             : 
     481             :         if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
     482             :                                            BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) ||
     483             :             !*remaining)
     484             :                 return;
     485             : 
     486             :         /* *remaining has already been aligned to 4 bytes, so *remaining >= 4 */
     487             : 
     488             :         /* init sock_ops */
     489             :         memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
     490             : 
     491             :         sock_ops.op = BPF_SOCK_OPS_HDR_OPT_LEN_CB;
     492             : 
     493             :         if (req) {
     494             :                 /* The listen "sk" cannot be passed here because
     495             :                  * it is not locked.  It would not make too much
     496             :                  * sense to do bpf_setsockopt(listen_sk) based
     497             :                  * on individual connection request also.
     498             :                  *
     499             :                  * Thus, "req" is passed here and the cgroup-bpf-progs
     500             :                  * of the listen "sk" will be run.
     501             :                  *
     502             :                  * "req" is also used here for fastopen even the "sk" here is
     503             :                  * a fullsock "child" sk.  It is to keep the behavior
     504             :                  * consistent between fastopen and non-fastopen on
     505             :                  * the bpf programming side.
     506             :                  */
     507             :                 sock_ops.sk = (struct sock *)req;
     508             :                 sock_ops.syn_skb = syn_skb;
     509             :         } else {
     510             :                 sock_owned_by_me(sk);
     511             : 
     512             :                 sock_ops.is_fullsock = 1;
     513             :                 sock_ops.sk = sk;
     514             :         }
     515             : 
     516             :         sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
     517             :         sock_ops.remaining_opt_len = *remaining;
     518             :         /* tcp_current_mss() does not pass a skb */
     519             :         if (skb)
     520             :                 bpf_skops_init_skb(&sock_ops, skb, 0);
     521             : 
     522             :         err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
     523             : 
     524             :         if (err || sock_ops.remaining_opt_len == *remaining)
     525             :                 return;
     526             : 
     527             :         opts->bpf_opt_len = *remaining - sock_ops.remaining_opt_len;
     528             :         /* round up to 4 bytes */
     529             :         opts->bpf_opt_len = (opts->bpf_opt_len + 3) & ~3;
     530             : 
     531             :         *remaining -= opts->bpf_opt_len;
     532             : }
     533             : 
     534             : static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
     535             :                                     struct request_sock *req,
     536             :                                     struct sk_buff *syn_skb,
     537             :                                     enum tcp_synack_type synack_type,
     538             :                                     struct tcp_out_options *opts)
     539             : {
     540             :         u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len;
     541             :         struct bpf_sock_ops_kern sock_ops;
     542             :         int err;
     543             : 
     544             :         if (likely(!max_opt_len))
     545             :                 return;
     546             : 
     547             :         memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
     548             : 
     549             :         sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
     550             : 
     551             :         if (req) {
     552             :                 sock_ops.sk = (struct sock *)req;
     553             :                 sock_ops.syn_skb = syn_skb;
     554             :         } else {
     555             :                 sock_owned_by_me(sk);
     556             : 
     557             :                 sock_ops.is_fullsock = 1;
     558             :                 sock_ops.sk = sk;
     559             :         }
     560             : 
     561             :         sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
     562             :         sock_ops.remaining_opt_len = max_opt_len;
     563             :         first_opt_off = tcp_hdrlen(skb) - max_opt_len;
     564             :         bpf_skops_init_skb(&sock_ops, skb, first_opt_off);
     565             : 
     566             :         err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
     567             : 
     568             :         if (err)
     569             :                 nr_written = 0;
     570             :         else
     571             :                 nr_written = max_opt_len - sock_ops.remaining_opt_len;
     572             : 
     573             :         if (nr_written < max_opt_len)
     574             :                 memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP,
     575             :                        max_opt_len - nr_written);
     576             : }
     577             : #else
     578           4 : static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
     579             :                                   struct request_sock *req,
     580             :                                   struct sk_buff *syn_skb,
     581             :                                   enum tcp_synack_type synack_type,
     582             :                                   struct tcp_out_options *opts,
     583             :                                   unsigned int *remaining)
     584             : {
     585           4 : }
     586             : 
     587         430 : static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
     588             :                                     struct request_sock *req,
     589             :                                     struct sk_buff *syn_skb,
     590             :                                     enum tcp_synack_type synack_type,
     591             :                                     struct tcp_out_options *opts)
     592             : {
     593         430 : }
     594             : #endif
     595             : 
     596             : /* Write previously computed TCP options to the packet.
     597             :  *
     598             :  * Beware: Something in the Internet is very sensitive to the ordering of
     599             :  * TCP options, we learned this through the hard way, so be careful here.
     600             :  * Luckily we can at least blame others for their non-compliance but from
     601             :  * inter-operability perspective it seems that we're somewhat stuck with
     602             :  * the ordering which we have been using if we want to keep working with
     603             :  * those broken things (not that it currently hurts anybody as there isn't
     604             :  * particular reason why the ordering would need to be changed).
     605             :  *
     606             :  * At least SACK_PERM as the first option is known to lead to a disaster
     607             :  * (but it may well be that other scenarios fail similarly).
     608             :  */
     609         430 : static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,
     610             :                               struct tcp_out_options *opts)
     611             : {
     612         430 :         u16 options = opts->options; /* mungable copy */
     613             : 
     614         430 :         if (unlikely(OPTION_MD5 & options)) {
     615           0 :                 *ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
     616             :                                (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
     617             :                 /* overload cookie hash location */
     618           0 :                 opts->hash_location = (__u8 *)ptr;
     619           0 :                 ptr += 4;
     620             :         }
     621             : 
     622         430 :         if (unlikely(opts->mss)) {
     623           4 :                 *ptr++ = htonl((TCPOPT_MSS << 24) |
     624             :                                (TCPOLEN_MSS << 16) |
     625             :                                opts->mss);
     626             :         }
     627             : 
     628         430 :         if (likely(OPTION_TS & options)) {
     629           0 :                 if (unlikely(OPTION_SACK_ADVERTISE & options)) {
     630           0 :                         *ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
     631             :                                        (TCPOLEN_SACK_PERM << 16) |
     632             :                                        (TCPOPT_TIMESTAMP << 8) |
     633             :                                        TCPOLEN_TIMESTAMP);
     634           0 :                         options &= ~OPTION_SACK_ADVERTISE;
     635             :                 } else {
     636           0 :                         *ptr++ = htonl((TCPOPT_NOP << 24) |
     637             :                                        (TCPOPT_NOP << 16) |
     638             :                                        (TCPOPT_TIMESTAMP << 8) |
     639             :                                        TCPOLEN_TIMESTAMP);
     640             :                 }
     641           0 :                 *ptr++ = htonl(opts->tsval);
     642           0 :                 *ptr++ = htonl(opts->tsecr);
     643             :         }
     644             : 
     645         430 :         if (unlikely(OPTION_SACK_ADVERTISE & options)) {
     646           0 :                 *ptr++ = htonl((TCPOPT_NOP << 24) |
     647             :                                (TCPOPT_NOP << 16) |
     648             :                                (TCPOPT_SACK_PERM << 8) |
     649             :                                TCPOLEN_SACK_PERM);
     650             :         }
     651             : 
     652         430 :         if (unlikely(OPTION_WSCALE & options)) {
     653           0 :                 *ptr++ = htonl((TCPOPT_NOP << 24) |
     654             :                                (TCPOPT_WINDOW << 16) |
     655             :                                (TCPOLEN_WINDOW << 8) |
     656             :                                opts->ws);
     657             :         }
     658             : 
     659         430 :         if (unlikely(opts->num_sack_blocks)) {
     660           0 :                 struct tcp_sack_block *sp = tp->rx_opt.dsack ?
     661           0 :                         tp->duplicate_sack : tp->selective_acks;
     662           0 :                 int this_sack;
     663             : 
     664           0 :                 *ptr++ = htonl((TCPOPT_NOP  << 24) |
     665             :                                (TCPOPT_NOP  << 16) |
     666             :                                (TCPOPT_SACK <<  8) |
     667             :                                (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
     668             :                                                      TCPOLEN_SACK_PERBLOCK)));
     669             : 
     670           0 :                 for (this_sack = 0; this_sack < opts->num_sack_blocks;
     671           0 :                      ++this_sack) {
     672           0 :                         *ptr++ = htonl(sp[this_sack].start_seq);
     673           0 :                         *ptr++ = htonl(sp[this_sack].end_seq);
     674             :                 }
     675             : 
     676           0 :                 tp->rx_opt.dsack = 0;
     677             :         }
     678             : 
     679         430 :         if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
     680           0 :                 struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
     681           0 :                 u8 *p = (u8 *)ptr;
     682           0 :                 u32 len; /* Fast Open option length */
     683             : 
     684           0 :                 if (foc->exp) {
     685           0 :                         len = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
     686           0 :                         *ptr = htonl((TCPOPT_EXP << 24) | (len << 16) |
     687             :                                      TCPOPT_FASTOPEN_MAGIC);
     688           0 :                         p += TCPOLEN_EXP_FASTOPEN_BASE;
     689             :                 } else {
     690           0 :                         len = TCPOLEN_FASTOPEN_BASE + foc->len;
     691           0 :                         *p++ = TCPOPT_FASTOPEN;
     692           0 :                         *p++ = len;
     693             :                 }
     694             : 
     695           0 :                 memcpy(p, foc->val, foc->len);
     696           0 :                 if ((len & 3) == 2) {
     697           0 :                         p[foc->len] = TCPOPT_NOP;
     698           0 :                         p[foc->len + 1] = TCPOPT_NOP;
     699             :                 }
     700           0 :                 ptr += (len + 3) >> 2;
     701             :         }
     702             : 
     703         430 :         smc_options_write(ptr, &options);
     704             : 
     705         430 :         mptcp_options_write(ptr, tp, opts);
     706         430 : }
     707             : 
     708           0 : static void smc_set_option(const struct tcp_sock *tp,
     709             :                            struct tcp_out_options *opts,
     710             :                            unsigned int *remaining)
     711             : {
     712             : #if IS_ENABLED(CONFIG_SMC)
     713             :         if (static_branch_unlikely(&tcp_have_smc)) {
     714             :                 if (tp->syn_smc) {
     715             :                         if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
     716             :                                 opts->options |= OPTION_SMC;
     717             :                                 *remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
     718             :                         }
     719             :                 }
     720             :         }
     721             : #endif
     722           0 : }
     723             : 
     724           4 : static void smc_set_option_cond(const struct tcp_sock *tp,
     725             :                                 const struct inet_request_sock *ireq,
     726             :                                 struct tcp_out_options *opts,
     727             :                                 unsigned int *remaining)
     728             : {
     729             : #if IS_ENABLED(CONFIG_SMC)
     730             :         if (static_branch_unlikely(&tcp_have_smc)) {
     731             :                 if (tp->syn_smc && ireq->smc_ok) {
     732             :                         if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
     733             :                                 opts->options |= OPTION_SMC;
     734             :                                 *remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
     735             :                         }
     736             :                 }
     737             :         }
     738             : #endif
     739           4 : }
     740             : 
     741           4 : static void mptcp_set_option_cond(const struct request_sock *req,
     742             :                                   struct tcp_out_options *opts,
     743             :                                   unsigned int *remaining)
     744             : {
     745           4 :         if (rsk_is_mptcp(req)) {
     746             :                 unsigned int size;
     747             : 
     748             :                 if (mptcp_synack_options(req, &size, &opts->mptcp)) {
     749             :                         if (*remaining >= size) {
     750             :                                 opts->options |= OPTION_MPTCP;
     751             :                                 *remaining -= size;
     752             :                         }
     753             :                 }
     754             :         }
     755             : }
     756             : 
     757             : /* Compute TCP options for SYN packets. This is not the final
     758             :  * network wire format yet.
     759             :  */
     760           0 : static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
     761             :                                 struct tcp_out_options *opts,
     762             :                                 struct tcp_md5sig_key **md5)
     763             : {
     764           0 :         struct tcp_sock *tp = tcp_sk(sk);
     765           0 :         unsigned int remaining = MAX_TCP_OPTION_SPACE;
     766           0 :         struct tcp_fastopen_request *fastopen = tp->fastopen_req;
     767             : 
     768           0 :         *md5 = NULL;
     769             : #ifdef CONFIG_TCP_MD5SIG
     770             :         if (static_branch_unlikely(&tcp_md5_needed) &&
     771             :             rcu_access_pointer(tp->md5sig_info)) {
     772             :                 *md5 = tp->af_specific->md5_lookup(sk, sk);
     773             :                 if (*md5) {
     774             :                         opts->options |= OPTION_MD5;
     775             :                         remaining -= TCPOLEN_MD5SIG_ALIGNED;
     776             :                 }
     777             :         }
     778             : #endif
     779             : 
     780             :         /* We always get an MSS option.  The option bytes which will be seen in
     781             :          * normal data packets should timestamps be used, must be in the MSS
     782             :          * advertised.  But we subtract them from tp->mss_cache so that
     783             :          * calculations in tcp_sendmsg are simpler etc.  So account for this
     784             :          * fact here if necessary.  If we don't do this correctly, as a
     785             :          * receiver we won't recognize data packets as being full sized when we
     786             :          * should, and thus we won't abide by the delayed ACK rules correctly.
     787             :          * SACKs don't matter, we never delay an ACK when we have any of those
     788             :          * going out.  */
     789           0 :         opts->mss = tcp_advertise_mss(sk);
     790           0 :         remaining -= TCPOLEN_MSS_ALIGNED;
     791             : 
     792           0 :         if (likely(sock_net(sk)->ipv4.sysctl_tcp_timestamps && !*md5)) {
     793           0 :                 opts->options |= OPTION_TS;
     794           0 :                 opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset;
     795           0 :                 opts->tsecr = tp->rx_opt.ts_recent;
     796           0 :                 remaining -= TCPOLEN_TSTAMP_ALIGNED;
     797             :         }
     798           0 :         if (likely(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) {
     799           0 :                 opts->ws = tp->rx_opt.rcv_wscale;
     800           0 :                 opts->options |= OPTION_WSCALE;
     801           0 :                 remaining -= TCPOLEN_WSCALE_ALIGNED;
     802             :         }
     803           0 :         if (likely(sock_net(sk)->ipv4.sysctl_tcp_sack)) {
     804           0 :                 opts->options |= OPTION_SACK_ADVERTISE;
     805           0 :                 if (unlikely(!(OPTION_TS & opts->options)))
     806           0 :                         remaining -= TCPOLEN_SACKPERM_ALIGNED;
     807             :         }
     808             : 
     809           0 :         if (fastopen && fastopen->cookie.len >= 0) {
     810           0 :                 u32 need = fastopen->cookie.len;
     811             : 
     812           0 :                 need += fastopen->cookie.exp ? TCPOLEN_EXP_FASTOPEN_BASE :
     813             :                                                TCPOLEN_FASTOPEN_BASE;
     814           0 :                 need = (need + 3) & ~3U;  /* Align to 32 bits */
     815           0 :                 if (remaining >= need) {
     816           0 :                         opts->options |= OPTION_FAST_OPEN_COOKIE;
     817           0 :                         opts->fastopen_cookie = &fastopen->cookie;
     818           0 :                         remaining -= need;
     819           0 :                         tp->syn_fastopen = 1;
     820           0 :                         tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0;
     821             :                 }
     822             :         }
     823             : 
     824           0 :         smc_set_option(tp, opts, &remaining);
     825             : 
     826           0 :         if (sk_is_mptcp(sk)) {
     827             :                 unsigned int size;
     828             : 
     829             :                 if (mptcp_syn_options(sk, skb, &size, &opts->mptcp)) {
     830             :                         opts->options |= OPTION_MPTCP;
     831             :                         remaining -= size;
     832             :                 }
     833             :         }
     834             : 
     835           0 :         bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
     836             : 
     837           0 :         return MAX_TCP_OPTION_SPACE - remaining;
     838             : }
     839             : 
     840             : /* Set up TCP options for SYN-ACKs. */
     841           4 : static unsigned int tcp_synack_options(const struct sock *sk,
     842             :                                        struct request_sock *req,
     843             :                                        unsigned int mss, struct sk_buff *skb,
     844             :                                        struct tcp_out_options *opts,
     845             :                                        const struct tcp_md5sig_key *md5,
     846             :                                        struct tcp_fastopen_cookie *foc,
     847             :                                        enum tcp_synack_type synack_type,
     848             :                                        struct sk_buff *syn_skb)
     849             : {
     850           4 :         struct inet_request_sock *ireq = inet_rsk(req);
     851           4 :         unsigned int remaining = MAX_TCP_OPTION_SPACE;
     852             : 
     853             : #ifdef CONFIG_TCP_MD5SIG
     854             :         if (md5) {
     855             :                 opts->options |= OPTION_MD5;
     856             :                 remaining -= TCPOLEN_MD5SIG_ALIGNED;
     857             : 
     858             :                 /* We can't fit any SACK blocks in a packet with MD5 + TS
     859             :                  * options. There was discussion about disabling SACK
     860             :                  * rather than TS in order to fit in better with old,
     861             :                  * buggy kernels, but that was deemed to be unnecessary.
     862             :                  */
     863             :                 if (synack_type != TCP_SYNACK_COOKIE)
     864             :                         ireq->tstamp_ok &= !ireq->sack_ok;
     865             :         }
     866             : #endif
     867             : 
     868             :         /* We always send an MSS option. */
     869           4 :         opts->mss = mss;
     870           4 :         remaining -= TCPOLEN_MSS_ALIGNED;
     871             : 
     872           4 :         if (likely(ireq->wscale_ok)) {
     873           0 :                 opts->ws = ireq->rcv_wscale;
     874           0 :                 opts->options |= OPTION_WSCALE;
     875           0 :                 remaining -= TCPOLEN_WSCALE_ALIGNED;
     876             :         }
     877           4 :         if (likely(ireq->tstamp_ok)) {
     878           0 :                 opts->options |= OPTION_TS;
     879           0 :                 opts->tsval = tcp_skb_timestamp(skb) + tcp_rsk(req)->ts_off;
     880           0 :                 opts->tsecr = req->ts_recent;
     881           0 :                 remaining -= TCPOLEN_TSTAMP_ALIGNED;
     882             :         }
     883           4 :         if (likely(ireq->sack_ok)) {
     884           0 :                 opts->options |= OPTION_SACK_ADVERTISE;
     885           0 :                 if (unlikely(!ireq->tstamp_ok))
     886           0 :                         remaining -= TCPOLEN_SACKPERM_ALIGNED;
     887             :         }
     888           4 :         if (foc != NULL && foc->len >= 0) {
     889           0 :                 u32 need = foc->len;
     890             : 
     891           0 :                 need += foc->exp ? TCPOLEN_EXP_FASTOPEN_BASE :
     892             :                                    TCPOLEN_FASTOPEN_BASE;
     893           0 :                 need = (need + 3) & ~3U;  /* Align to 32 bits */
     894           0 :                 if (remaining >= need) {
     895           0 :                         opts->options |= OPTION_FAST_OPEN_COOKIE;
     896           0 :                         opts->fastopen_cookie = foc;
     897           0 :                         remaining -= need;
     898             :                 }
     899             :         }
     900             : 
     901           4 :         mptcp_set_option_cond(req, opts, &remaining);
     902             : 
     903           4 :         smc_set_option_cond(tcp_sk(sk), ireq, opts, &remaining);
     904             : 
     905           4 :         bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
     906             :                               synack_type, opts, &remaining);
     907             : 
     908           4 :         return MAX_TCP_OPTION_SPACE - remaining;
     909             : }
     910             : 
     911             : /* Compute TCP options for ESTABLISHED sockets. This is not the
     912             :  * final wire format yet.
     913             :  */
     914         922 : static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
     915             :                                         struct tcp_out_options *opts,
     916             :                                         struct tcp_md5sig_key **md5)
     917             : {
     918         922 :         struct tcp_sock *tp = tcp_sk(sk);
     919         922 :         unsigned int size = 0;
     920         922 :         unsigned int eff_sacks;
     921             : 
     922         922 :         opts->options = 0;
     923             : 
     924         922 :         *md5 = NULL;
     925             : #ifdef CONFIG_TCP_MD5SIG
     926             :         if (static_branch_unlikely(&tcp_md5_needed) &&
     927             :             rcu_access_pointer(tp->md5sig_info)) {
     928             :                 *md5 = tp->af_specific->md5_lookup(sk, sk);
     929             :                 if (*md5) {
     930             :                         opts->options |= OPTION_MD5;
     931             :                         size += TCPOLEN_MD5SIG_ALIGNED;
     932             :                 }
     933             :         }
     934             : #endif
     935             : 
     936         922 :         if (likely(tp->rx_opt.tstamp_ok)) {
     937           0 :                 opts->options |= OPTION_TS;
     938           0 :                 opts->tsval = skb ? tcp_skb_timestamp(skb) + tp->tsoffset : 0;
     939           0 :                 opts->tsecr = tp->rx_opt.ts_recent;
     940           0 :                 size += TCPOLEN_TSTAMP_ALIGNED;
     941             :         }
     942             : 
     943             :         /* MPTCP options have precedence over SACK for the limited TCP
     944             :          * option space because a MPTCP connection would be forced to
     945             :          * fall back to regular TCP if a required multipath option is
     946             :          * missing. SACK still gets a chance to use whatever space is
     947             :          * left.
     948             :          */
     949         922 :         if (sk_is_mptcp(sk)) {
     950             :                 unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
     951             :                 unsigned int opt_size = 0;
     952             : 
     953             :                 if (mptcp_established_options(sk, skb, &opt_size, remaining,
     954             :                                               &opts->mptcp)) {
     955             :                         opts->options |= OPTION_MPTCP;
     956             :                         size += opt_size;
     957             :                 }
     958             :         }
     959             : 
     960         922 :         eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
     961         922 :         if (unlikely(eff_sacks)) {
     962           0 :                 const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
     963           0 :                 if (unlikely(remaining < TCPOLEN_SACK_BASE_ALIGNED +
     964             :                                          TCPOLEN_SACK_PERBLOCK))
     965             :                         return size;
     966             : 
     967           0 :                 opts->num_sack_blocks =
     968           0 :                         min_t(unsigned int, eff_sacks,
     969             :                               (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
     970             :                               TCPOLEN_SACK_PERBLOCK);
     971             : 
     972           0 :                 size += TCPOLEN_SACK_BASE_ALIGNED +
     973           0 :                         opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
     974             :         }
     975             : 
     976         922 :         if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp,
     977             :                                             BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) {
     978             :                 unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
     979             : 
     980             :                 bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
     981             : 
     982             :                 size = MAX_TCP_OPTION_SPACE - remaining;
     983             :         }
     984             : 
     985         922 :         return size;
     986             : }
     987             : 
     988             : 
     989             : /* TCP SMALL QUEUES (TSQ)
     990             :  *
     991             :  * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev)
     992             :  * to reduce RTT and bufferbloat.
     993             :  * We do this using a special skb destructor (tcp_wfree).
     994             :  *
     995             :  * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb
     996             :  * needs to be reallocated in a driver.
     997             :  * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc
     998             :  *
     999             :  * Since transmit from skb destructor is forbidden, we use a tasklet
    1000             :  * to process all sockets that eventually need to send more skbs.
    1001             :  * We use one tasklet per cpu, with its own queue of sockets.
    1002             :  */
    1003             : struct tsq_tasklet {
    1004             :         struct tasklet_struct   tasklet;
    1005             :         struct list_head        head; /* queue of tcp sockets */
    1006             : };
    1007             : static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet);
    1008             : 
    1009          58 : static void tcp_tsq_write(struct sock *sk)
    1010             : {
    1011          58 :         if ((1 << sk->sk_state) &
    1012             :             (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
    1013             :              TCPF_CLOSE_WAIT  | TCPF_LAST_ACK)) {
    1014          58 :                 struct tcp_sock *tp = tcp_sk(sk);
    1015             : 
    1016          58 :                 if (tp->lost_out > tp->retrans_out &&
    1017           0 :                     tp->snd_cwnd > tcp_packets_in_flight(tp)) {
    1018           0 :                         tcp_mstamp_refresh(tp);
    1019           0 :                         tcp_xmit_retransmit_queue(sk);
    1020             :                 }
    1021             : 
    1022          58 :                 tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle,
    1023             :                                0, GFP_ATOMIC);
    1024             :         }
    1025          58 : }
    1026             : 
    1027          58 : static void tcp_tsq_handler(struct sock *sk)
    1028             : {
    1029          58 :         bh_lock_sock(sk);
    1030          58 :         if (!sock_owned_by_user(sk))
    1031          18 :                 tcp_tsq_write(sk);
    1032          40 :         else if (!test_and_set_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
    1033          40 :                 sock_hold(sk);
    1034          58 :         bh_unlock_sock(sk);
    1035          58 : }
    1036             : /*
    1037             :  * One tasklet per cpu tries to send more skbs.
    1038             :  * We run in tasklet context but need to disable irqs when
    1039             :  * transferring tsq->head because tcp_wfree() might
    1040             :  * interrupt us (non NAPI drivers)
    1041             :  */
    1042          58 : static void tcp_tasklet_func(struct tasklet_struct *t)
    1043             : {
    1044          58 :         struct tsq_tasklet *tsq = from_tasklet(tsq,  t, tasklet);
    1045          58 :         LIST_HEAD(list);
    1046          58 :         unsigned long flags;
    1047          58 :         struct list_head *q, *n;
    1048          58 :         struct tcp_sock *tp;
    1049          58 :         struct sock *sk;
    1050             : 
    1051         116 :         local_irq_save(flags);
    1052          58 :         list_splice_init(&tsq->head, &list);
    1053          58 :         local_irq_restore(flags);
    1054             : 
    1055         116 :         list_for_each_safe(q, n, &list) {
    1056          58 :                 tp = list_entry(q, struct tcp_sock, tsq_node);
    1057          58 :                 list_del(&tp->tsq_node);
    1058             : 
    1059          58 :                 sk = (struct sock *)tp;
    1060          58 :                 smp_mb__before_atomic();
    1061          58 :                 clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
    1062             : 
    1063          58 :                 tcp_tsq_handler(sk);
    1064          58 :                 sk_free(sk);
    1065             :         }
    1066          58 : }
    1067             : 
    1068             : #define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED |           \
    1069             :                           TCPF_WRITE_TIMER_DEFERRED |   \
    1070             :                           TCPF_DELACK_TIMER_DEFERRED |  \
    1071             :                           TCPF_MTU_REDUCED_DEFERRED)
    1072             : /**
    1073             :  * tcp_release_cb - tcp release_sock() callback
    1074             :  * @sk: socket
    1075             :  *
    1076             :  * called from release_sock() to perform protocol dependent
    1077             :  * actions before socket release.
    1078             :  */
    1079         651 : void tcp_release_cb(struct sock *sk)
    1080             : {
    1081         651 :         unsigned long flags, nflags;
    1082             : 
    1083             :         /* perform an atomic operation only if at least one flag is set */
    1084         651 :         do {
    1085         651 :                 flags = sk->sk_tsq_flags;
    1086         651 :                 if (!(flags & TCP_DEFERRED_ALL))
    1087             :                         return;
    1088          40 :                 nflags = flags & ~TCP_DEFERRED_ALL;
    1089          40 :         } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
    1090             : 
    1091          40 :         if (flags & TCPF_TSQ_DEFERRED) {
    1092          40 :                 tcp_tsq_write(sk);
    1093          40 :                 __sock_put(sk);
    1094             :         }
    1095             :         /* Here begins the tricky part :
    1096             :          * We are called from release_sock() with :
    1097             :          * 1) BH disabled
    1098             :          * 2) sk_lock.slock spinlock held
    1099             :          * 3) socket owned by us (sk->sk_lock.owned == 1)
    1100             :          *
    1101             :          * But following code is meant to be called from BH handlers,
    1102             :          * so we should keep BH disabled, but early release socket ownership
    1103             :          */
    1104          40 :         sock_release_ownership(sk);
    1105             : 
    1106          40 :         if (flags & TCPF_WRITE_TIMER_DEFERRED) {
    1107           0 :                 tcp_write_timer_handler(sk);
    1108           0 :                 __sock_put(sk);
    1109             :         }
    1110          40 :         if (flags & TCPF_DELACK_TIMER_DEFERRED) {
    1111           0 :                 tcp_delack_timer_handler(sk);
    1112           0 :                 __sock_put(sk);
    1113             :         }
    1114          40 :         if (flags & TCPF_MTU_REDUCED_DEFERRED) {
    1115           0 :                 inet_csk(sk)->icsk_af_ops->mtu_reduced(sk);
    1116           0 :                 __sock_put(sk);
    1117             :         }
    1118             : }
    1119             : EXPORT_SYMBOL(tcp_release_cb);
    1120             : 
    1121           1 : void __init tcp_tasklet_init(void)
    1122             : {
    1123           1 :         int i;
    1124             : 
    1125           5 :         for_each_possible_cpu(i) {
    1126           4 :                 struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i);
    1127             : 
    1128           4 :                 INIT_LIST_HEAD(&tsq->head);
    1129           4 :                 tasklet_setup(&tsq->tasklet, tcp_tasklet_func);
    1130             :         }
    1131           1 : }
    1132             : 
    1133             : /*
    1134             :  * Write buffer destructor automatically called from kfree_skb.
    1135             :  * We can't xmit new skbs from this context, as we might already
    1136             :  * hold qdisc lock.
    1137             :  */
    1138         364 : void tcp_wfree(struct sk_buff *skb)
    1139             : {
    1140         364 :         struct sock *sk = skb->sk;
    1141         364 :         struct tcp_sock *tp = tcp_sk(sk);
    1142         364 :         unsigned long flags, nval, oval;
    1143             : 
    1144             :         /* Keep one reference on sk_wmem_alloc.
    1145             :          * Will be released by sk_free() from here or tcp_tasklet_func()
    1146             :          */
    1147         364 :         WARN_ON(refcount_sub_and_test(skb->truesize - 1, &sk->sk_wmem_alloc));
    1148             : 
    1149             :         /* If this softirq is serviced by ksoftirqd, we are likely under stress.
    1150             :          * Wait until our queues (qdisc + devices) are drained.
    1151             :          * This gives :
    1152             :          * - less callbacks to tcp_write_xmit(), reducing stress (batches)
    1153             :          * - chance for incoming ACK (processed by another cpu maybe)
    1154             :          *   to migrate this flow (skb->ooo_okay will be eventually set)
    1155             :          */
    1156         403 :         if (refcount_read(&sk->sk_wmem_alloc) >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
    1157           5 :                 goto out;
    1158             : 
    1159         359 :         for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
    1160         359 :                 struct tsq_tasklet *tsq;
    1161         359 :                 bool empty;
    1162             : 
    1163         359 :                 if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
    1164         301 :                         goto out;
    1165             : 
    1166          58 :                 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED;
    1167          58 :                 nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
    1168          58 :                 if (nval != oval)
    1169           0 :                         continue;
    1170             : 
    1171             :                 /* queue this socket to tasklet queue */
    1172         116 :                 local_irq_save(flags);
    1173          58 :                 tsq = this_cpu_ptr(&tsq_tasklet);
    1174          58 :                 empty = list_empty(&tsq->head);
    1175          58 :                 list_add(&tp->tsq_node, &tsq->head);
    1176          58 :                 if (empty)
    1177          58 :                         tasklet_schedule(&tsq->tasklet);
    1178          58 :                 local_irq_restore(flags);
    1179             :                 return;
    1180             :         }
    1181         306 : out:
    1182         306 :         sk_free(sk);
    1183             : }
    1184             : 
    1185             : /* Note: Called under soft irq.
    1186             :  * We can call TCP stack right away, unless socket is owned by user.
    1187             :  */
    1188           0 : enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer)
    1189             : {
    1190           0 :         struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer);
    1191           0 :         struct sock *sk = (struct sock *)tp;
    1192             : 
    1193           0 :         tcp_tsq_handler(sk);
    1194           0 :         sock_put(sk);
    1195             : 
    1196           0 :         return HRTIMER_NORESTART;
    1197             : }
    1198             : 
    1199         364 : static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb,
    1200             :                                       u64 prior_wstamp)
    1201             : {
    1202         364 :         struct tcp_sock *tp = tcp_sk(sk);
    1203             : 
    1204         364 :         if (sk->sk_pacing_status != SK_PACING_NONE) {
    1205           0 :                 unsigned long rate = sk->sk_pacing_rate;
    1206             : 
    1207             :                 /* Original sch_fq does not pace first 10 MSS
    1208             :                  * Note that tp->data_segs_out overflows after 2^32 packets,
    1209             :                  * this is a minor annoyance.
    1210             :                  */
    1211           0 :                 if (rate != ~0UL && rate && tp->data_segs_out >= 10) {
    1212           0 :                         u64 len_ns = div64_ul((u64)skb->len * NSEC_PER_SEC, rate);
    1213           0 :                         u64 credit = tp->tcp_wstamp_ns - prior_wstamp;
    1214             : 
    1215             :                         /* take into account OS jitter */
    1216           0 :                         len_ns -= min_t(u64, len_ns / 2, credit);
    1217           0 :                         tp->tcp_wstamp_ns += len_ns;
    1218             :                 }
    1219             :         }
    1220         364 :         list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
    1221         364 : }
    1222             : 
    1223             : INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
    1224             : INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
    1225             : INDIRECT_CALLABLE_DECLARE(void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb));
    1226             : 
    1227             : /* This routine actually transmits TCP packets queued in by
    1228             :  * tcp_do_sendmsg().  This is used by both the initial
    1229             :  * transmission and possible later retransmissions.
    1230             :  * All SKB's seen here are completely headerless.  It is our
    1231             :  * job to build the TCP header, and pass the packet down to
    1232             :  * IP so it can do the same plus pass the packet off to the
    1233             :  * device.
    1234             :  *
    1235             :  * We are working here with either a clone of the original
    1236             :  * SKB, or a fresh unique copy made by the retransmit engine.
    1237             :  */
    1238         426 : static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
    1239             :                               int clone_it, gfp_t gfp_mask, u32 rcv_nxt)
    1240             : {
    1241         426 :         const struct inet_connection_sock *icsk = inet_csk(sk);
    1242         426 :         struct inet_sock *inet;
    1243         426 :         struct tcp_sock *tp;
    1244         426 :         struct tcp_skb_cb *tcb;
    1245         426 :         struct tcp_out_options opts;
    1246         426 :         unsigned int tcp_options_size, tcp_header_size;
    1247         426 :         struct sk_buff *oskb = NULL;
    1248         426 :         struct tcp_md5sig_key *md5;
    1249         426 :         struct tcphdr *th;
    1250         426 :         u64 prior_wstamp;
    1251         426 :         int err;
    1252             : 
    1253         426 :         BUG_ON(!skb || !tcp_skb_pcount(skb));
    1254         426 :         tp = tcp_sk(sk);
    1255         426 :         prior_wstamp = tp->tcp_wstamp_ns;
    1256         426 :         tp->tcp_wstamp_ns = max(tp->tcp_wstamp_ns, tp->tcp_clock_cache);
    1257         426 :         skb->skb_mstamp_ns = tp->tcp_wstamp_ns;
    1258         426 :         if (clone_it) {
    1259         364 :                 TCP_SKB_CB(skb)->tx.in_flight = TCP_SKB_CB(skb)->end_seq
    1260         364 :                         - tp->snd_una;
    1261         364 :                 oskb = skb;
    1262             : 
    1263         364 :                 tcp_skb_tsorted_save(oskb) {
    1264         364 :                         if (unlikely(skb_cloned(oskb)))
    1265           0 :                                 skb = pskb_copy(oskb, gfp_mask);
    1266             :                         else
    1267         364 :                                 skb = skb_clone(oskb, gfp_mask);
    1268         364 :                 } tcp_skb_tsorted_restore(oskb);
    1269             : 
    1270         364 :                 if (unlikely(!skb))
    1271             :                         return -ENOBUFS;
    1272             :                 /* retransmit skbs might have a non zero value in skb->dev
    1273             :                  * because skb->dev is aliased with skb->rbnode.rb_left
    1274             :                  */
    1275         364 :                 skb->dev = NULL;
    1276             :         }
    1277             : 
    1278         426 :         inet = inet_sk(sk);
    1279         426 :         tcb = TCP_SKB_CB(skb);
    1280         426 :         memset(&opts, 0, sizeof(opts));
    1281             : 
    1282         426 :         if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
    1283           0 :                 tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
    1284             :         } else {
    1285         426 :                 tcp_options_size = tcp_established_options(sk, skb, &opts,
    1286             :                                                            &md5);
    1287             :                 /* Force a PSH flag on all (GSO) packets to expedite GRO flush
    1288             :                  * at receiver : This slightly improve GRO performance.
    1289             :                  * Note that we do not force the PSH flag for non GSO packets,
    1290             :                  * because they might be sent under high congestion events,
    1291             :                  * and in this case it is better to delay the delivery of 1-MSS
    1292             :                  * packets and thus the corresponding ACK packet that would
    1293             :                  * release the following packet.
    1294             :                  */
    1295         426 :                 if (tcp_skb_pcount(skb) > 1)
    1296           0 :                         tcb->tcp_flags |= TCPHDR_PSH;
    1297             :         }
    1298         426 :         tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
    1299             : 
    1300             :         /* if no packet is in qdisc/device queue, then allow XPS to select
    1301             :          * another queue. We can be called from tcp_tsq_handler()
    1302             :          * which holds one reference to sk.
    1303             :          *
    1304             :          * TODO: Ideally, in-flight pure ACK packets should not matter here.
    1305             :          * One way to get this would be to set skb->truesize = 2 on them.
    1306             :          */
    1307         426 :         skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1);
    1308             : 
    1309             :         /* If we had to use memory reserve to allocate this skb,
    1310             :          * this might cause drops if packet is looped back :
    1311             :          * Other socket might not have SOCK_MEMALLOC.
    1312             :          * Packets not looped back do not care about pfmemalloc.
    1313             :          */
    1314         426 :         skb->pfmemalloc = 0;
    1315             : 
    1316         426 :         skb_push(skb, tcp_header_size);
    1317         426 :         skb_reset_transport_header(skb);
    1318             : 
    1319         426 :         skb_orphan(skb);
    1320         426 :         skb->sk = sk;
    1321         426 :         skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
    1322         426 :         refcount_add(skb->truesize, &sk->sk_wmem_alloc);
    1323             : 
    1324         426 :         skb_set_dst_pending_confirm(skb, sk->sk_dst_pending_confirm);
    1325             : 
    1326             :         /* Build TCP header and checksum it. */
    1327         426 :         th = (struct tcphdr *)skb->data;
    1328         426 :         th->source           = inet->inet_sport;
    1329         426 :         th->dest             = inet->inet_dport;
    1330         426 :         th->seq                      = htonl(tcb->seq);
    1331         426 :         th->ack_seq          = htonl(rcv_nxt);
    1332         426 :         *(((__be16 *)th) + 6)   = htons(((tcp_header_size >> 2) << 12) |
    1333             :                                         tcb->tcp_flags);
    1334             : 
    1335         426 :         th->check            = 0;
    1336         426 :         th->urg_ptr          = 0;
    1337             : 
    1338             :         /* The urg_mode check is necessary during a below snd_una win probe */
    1339         426 :         if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
    1340           0 :                 if (before(tp->snd_up, tcb->seq + 0x10000)) {
    1341           0 :                         th->urg_ptr = htons(tp->snd_up - tcb->seq);
    1342           0 :                         th->urg = 1;
    1343           0 :                 } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
    1344           0 :                         th->urg_ptr = htons(0xFFFF);
    1345           0 :                         th->urg = 1;
    1346             :                 }
    1347             :         }
    1348             : 
    1349         426 :         skb_shinfo(skb)->gso_type = sk->sk_gso_type;
    1350         426 :         if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
    1351         426 :                 th->window      = htons(tcp_select_window(sk));
    1352         426 :                 tcp_ecn_send(sk, skb, th, tcp_header_size);
    1353             :         } else {
    1354             :                 /* RFC1323: The window in SYN & SYN/ACK segments
    1355             :                  * is never scaled.
    1356             :                  */
    1357           0 :                 th->window   = htons(min(tp->rcv_wnd, 65535U));
    1358             :         }
    1359             : 
    1360         426 :         tcp_options_write((__be32 *)(th + 1), tp, &opts);
    1361             : 
    1362             : #ifdef CONFIG_TCP_MD5SIG
    1363             :         /* Calculate the MD5 hash, as we have all we need now */
    1364             :         if (md5) {
    1365             :                 sk_nocaps_add(sk, NETIF_F_GSO_MASK);
    1366             :                 tp->af_specific->calc_md5_hash(opts.hash_location,
    1367             :                                                md5, sk, skb);
    1368             :         }
    1369             : #endif
    1370             : 
    1371             :         /* BPF prog is the last one writing header option */
    1372         426 :         bpf_skops_write_hdr_opt(sk, skb, NULL, NULL, 0, &opts);
    1373             : 
    1374         426 :         INDIRECT_CALL_INET(icsk->icsk_af_ops->send_check,
    1375             :                            tcp_v6_send_check, tcp_v4_send_check,
    1376             :                            sk, skb);
    1377             : 
    1378         426 :         if (likely(tcb->tcp_flags & TCPHDR_ACK))
    1379         426 :                 tcp_event_ack_sent(sk, tcp_skb_pcount(skb), rcv_nxt);
    1380             : 
    1381         426 :         if (skb->len != tcp_header_size) {
    1382         361 :                 tcp_event_data_sent(tp, sk);
    1383         361 :                 tp->data_segs_out += tcp_skb_pcount(skb);
    1384         361 :                 tp->bytes_sent += skb->len - tcp_header_size;
    1385             :         }
    1386             : 
    1387         426 :         if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
    1388         426 :                 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
    1389             :                               tcp_skb_pcount(skb));
    1390             : 
    1391         426 :         tp->segs_out += tcp_skb_pcount(skb);
    1392         426 :         skb_set_hash_from_sk(skb, sk);
    1393             :         /* OK, its time to fill skb_shinfo(skb)->gso_{segs|size} */
    1394         426 :         skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb);
    1395         426 :         skb_shinfo(skb)->gso_size = tcp_skb_mss(skb);
    1396             : 
    1397             :         /* Leave earliest departure time in skb->tstamp (skb->skb_mstamp_ns) */
    1398             : 
    1399             :         /* Cleanup our debris for IP stacks */
    1400         426 :         memset(skb->cb, 0, max(sizeof(struct inet_skb_parm),
    1401             :                                sizeof(struct inet6_skb_parm)));
    1402             : 
    1403         426 :         tcp_add_tx_delay(skb, tp);
    1404             : 
    1405         426 :         err = INDIRECT_CALL_INET(icsk->icsk_af_ops->queue_xmit,
    1406             :                                  inet6_csk_xmit, ip_queue_xmit,
    1407             :                                  sk, skb, &inet->cork.fl);
    1408             : 
    1409         426 :         if (unlikely(err > 0)) {
    1410           0 :                 tcp_enter_cwr(sk);
    1411           0 :                 err = net_xmit_eval(err);
    1412             :         }
    1413         426 :         if (!err && oskb) {
    1414         364 :                 tcp_update_skb_after_send(sk, oskb, prior_wstamp);
    1415         364 :                 tcp_rate_skb_sent(sk, oskb);
    1416             :         }
    1417             :         return err;
    1418             : }
    1419             : 
    1420         364 : static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
    1421             :                             gfp_t gfp_mask)
    1422             : {
    1423         364 :         return __tcp_transmit_skb(sk, skb, clone_it, gfp_mask,
    1424         364 :                                   tcp_sk(sk)->rcv_nxt);
    1425             : }
    1426             : 
    1427             : /* This routine just queues the buffer for sending.
    1428             :  *
    1429             :  * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
    1430             :  * otherwise socket can stall.
    1431             :  */
    1432           3 : static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
    1433             : {
    1434           3 :         struct tcp_sock *tp = tcp_sk(sk);
    1435             : 
    1436             :         /* Advance write_seq and place onto the write_queue. */
    1437           3 :         WRITE_ONCE(tp->write_seq, TCP_SKB_CB(skb)->end_seq);
    1438           3 :         __skb_header_release(skb);
    1439           3 :         tcp_add_write_queue_tail(sk, skb);
    1440           3 :         sk_wmem_queued_add(sk, skb->truesize);
    1441           3 :         sk_mem_charge(sk, skb->truesize);
    1442           3 : }
    1443             : 
    1444             : /* Initialize TSO segments for a packet. */
    1445         361 : static void tcp_set_skb_tso_segs(struct sk_buff *skb, unsigned int mss_now)
    1446             : {
    1447           0 :         if (skb->len <= mss_now) {
    1448             :                 /* Avoid the costly divide in the normal
    1449             :                  * non-TSO case.
    1450             :                  */
    1451         361 :                 tcp_skb_pcount_set(skb, 1);
    1452         361 :                 TCP_SKB_CB(skb)->tcp_gso_size = 0;
    1453             :         } else {
    1454           0 :                 tcp_skb_pcount_set(skb, DIV_ROUND_UP(skb->len, mss_now));
    1455           0 :                 TCP_SKB_CB(skb)->tcp_gso_size = mss_now;
    1456             :         }
    1457           0 : }
    1458             : 
    1459             : /* Pcount in the middle of the write queue got changed, we need to do various
    1460             :  * tweaks to fix counters
    1461             :  */
    1462           0 : static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
    1463             : {
    1464           0 :         struct tcp_sock *tp = tcp_sk(sk);
    1465             : 
    1466           0 :         tp->packets_out -= decr;
    1467             : 
    1468           0 :         if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
    1469           0 :                 tp->sacked_out -= decr;
    1470           0 :         if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
    1471           0 :                 tp->retrans_out -= decr;
    1472           0 :         if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
    1473           0 :                 tp->lost_out -= decr;
    1474             : 
    1475             :         /* Reno case is special. Sigh... */
    1476           0 :         if (tcp_is_reno(tp) && decr > 0)
    1477           0 :                 tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
    1478             : 
    1479           0 :         if (tp->lost_skb_hint &&
    1480           0 :             before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
    1481           0 :             (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
    1482           0 :                 tp->lost_cnt_hint -= decr;
    1483             : 
    1484           0 :         tcp_verify_left_out(tp);
    1485           0 : }
    1486             : 
    1487           0 : static bool tcp_has_tx_tstamp(const struct sk_buff *skb)
    1488             : {
    1489           0 :         return TCP_SKB_CB(skb)->txstamp_ack ||
    1490           0 :                 (skb_shinfo(skb)->tx_flags & SKBTX_ANY_TSTAMP);
    1491             : }
    1492             : 
    1493           0 : static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2)
    1494             : {
    1495           0 :         struct skb_shared_info *shinfo = skb_shinfo(skb);
    1496             : 
    1497           0 :         if (unlikely(tcp_has_tx_tstamp(skb)) &&
    1498           0 :             !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) {
    1499           0 :                 struct skb_shared_info *shinfo2 = skb_shinfo(skb2);
    1500           0 :                 u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP;
    1501             : 
    1502           0 :                 shinfo->tx_flags &= ~tsflags;
    1503           0 :                 shinfo2->tx_flags |= tsflags;
    1504           0 :                 swap(shinfo->tskey, shinfo2->tskey);
    1505           0 :                 TCP_SKB_CB(skb2)->txstamp_ack = TCP_SKB_CB(skb)->txstamp_ack;
    1506           0 :                 TCP_SKB_CB(skb)->txstamp_ack = 0;
    1507             :         }
    1508           0 : }
    1509             : 
    1510           0 : static void tcp_skb_fragment_eor(struct sk_buff *skb, struct sk_buff *skb2)
    1511             : {
    1512           0 :         TCP_SKB_CB(skb2)->eor = TCP_SKB_CB(skb)->eor;
    1513           0 :         TCP_SKB_CB(skb)->eor = 0;
    1514             : }
    1515             : 
    1516             : /* Insert buff after skb on the write or rtx queue of sk.  */
    1517           0 : static void tcp_insert_write_queue_after(struct sk_buff *skb,
    1518             :                                          struct sk_buff *buff,
    1519             :                                          struct sock *sk,
    1520             :                                          enum tcp_queue tcp_queue)
    1521             : {
    1522           0 :         if (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE)
    1523           0 :                 __skb_queue_after(&sk->sk_write_queue, skb, buff);
    1524             :         else
    1525           0 :                 tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
    1526           0 : }
    1527             : 
    1528             : /* Function to create two new TCP segments.  Shrinks the given segment
    1529             :  * to the specified size and appends a new segment with the rest of the
    1530             :  * packet to the list.  This won't be called frequently, I hope.
    1531             :  * Remember, these are still headerless SKBs at this point.
    1532             :  */
    1533           0 : int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
    1534             :                  struct sk_buff *skb, u32 len,
    1535             :                  unsigned int mss_now, gfp_t gfp)
    1536             : {
    1537           0 :         struct tcp_sock *tp = tcp_sk(sk);
    1538           0 :         struct sk_buff *buff;
    1539           0 :         int nsize, old_factor;
    1540           0 :         long limit;
    1541           0 :         int nlen;
    1542           0 :         u8 flags;
    1543             : 
    1544           0 :         if (WARN_ON(len > skb->len))
    1545             :                 return -EINVAL;
    1546             : 
    1547           0 :         nsize = skb_headlen(skb) - len;
    1548           0 :         if (nsize < 0)
    1549             :                 nsize = 0;
    1550             : 
    1551             :         /* tcp_sendmsg() can overshoot sk_wmem_queued by one full size skb.
    1552             :          * We need some allowance to not penalize applications setting small
    1553             :          * SO_SNDBUF values.
    1554             :          * Also allow first and last skb in retransmit queue to be split.
    1555             :          */
    1556           0 :         limit = sk->sk_sndbuf + 2 * SKB_TRUESIZE(GSO_MAX_SIZE);
    1557           0 :         if (unlikely((sk->sk_wmem_queued >> 1) > limit &&
    1558             :                      tcp_queue != TCP_FRAG_IN_WRITE_QUEUE &&
    1559             :                      skb != tcp_rtx_queue_head(sk) &&
    1560             :                      skb != tcp_rtx_queue_tail(sk))) {
    1561           0 :                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPWQUEUETOOBIG);
    1562           0 :                 return -ENOMEM;
    1563             :         }
    1564             : 
    1565           0 :         if (skb_unclone(skb, gfp))
    1566             :                 return -ENOMEM;
    1567             : 
    1568             :         /* Get a new skb... force flag on. */
    1569           0 :         buff = sk_stream_alloc_skb(sk, nsize, gfp, true);
    1570           0 :         if (!buff)
    1571             :                 return -ENOMEM; /* We'll just try again later. */
    1572           0 :         skb_copy_decrypted(buff, skb);
    1573           0 :         mptcp_skb_ext_copy(buff, skb);
    1574             : 
    1575           0 :         sk_wmem_queued_add(sk, buff->truesize);
    1576           0 :         sk_mem_charge(sk, buff->truesize);
    1577           0 :         nlen = skb->len - len - nsize;
    1578           0 :         buff->truesize += nlen;
    1579           0 :         skb->truesize -= nlen;
    1580             : 
    1581             :         /* Correct the sequence numbers. */
    1582           0 :         TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
    1583           0 :         TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
    1584           0 :         TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
    1585             : 
    1586             :         /* PSH and FIN should only be set in the second packet. */
    1587           0 :         flags = TCP_SKB_CB(skb)->tcp_flags;
    1588           0 :         TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
    1589           0 :         TCP_SKB_CB(buff)->tcp_flags = flags;
    1590           0 :         TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
    1591           0 :         tcp_skb_fragment_eor(skb, buff);
    1592             : 
    1593           0 :         skb_split(skb, buff, len);
    1594             : 
    1595           0 :         buff->ip_summed = CHECKSUM_PARTIAL;
    1596             : 
    1597           0 :         buff->tstamp = skb->tstamp;
    1598           0 :         tcp_fragment_tstamp(skb, buff);
    1599             : 
    1600           0 :         old_factor = tcp_skb_pcount(skb);
    1601             : 
    1602             :         /* Fix up tso_factor for both original and new SKB.  */
    1603           0 :         tcp_set_skb_tso_segs(skb, mss_now);
    1604           0 :         tcp_set_skb_tso_segs(buff, mss_now);
    1605             : 
    1606             :         /* Update delivered info for the new segment */
    1607           0 :         TCP_SKB_CB(buff)->tx = TCP_SKB_CB(skb)->tx;
    1608             : 
    1609             :         /* If this packet has been sent out already, we must
    1610             :          * adjust the various packet counters.
    1611             :          */
    1612           0 :         if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
    1613           0 :                 int diff = old_factor - tcp_skb_pcount(skb) -
    1614           0 :                         tcp_skb_pcount(buff);
    1615             : 
    1616           0 :                 if (diff)
    1617           0 :                         tcp_adjust_pcount(sk, skb, diff);
    1618             :         }
    1619             : 
    1620             :         /* Link BUFF into the send queue. */
    1621           0 :         __skb_header_release(buff);
    1622           0 :         tcp_insert_write_queue_after(skb, buff, sk, tcp_queue);
    1623           0 :         if (tcp_queue == TCP_FRAG_IN_RTX_QUEUE)
    1624           0 :                 list_add(&buff->tcp_tsorted_anchor, &skb->tcp_tsorted_anchor);
    1625             : 
    1626             :         return 0;
    1627             : }
    1628             : 
    1629             : /* This is similar to __pskb_pull_tail(). The difference is that pulled
    1630             :  * data is not copied, but immediately discarded.
    1631             :  */
    1632           0 : static int __pskb_trim_head(struct sk_buff *skb, int len)
    1633             : {
    1634           0 :         struct skb_shared_info *shinfo;
    1635           0 :         int i, k, eat;
    1636             : 
    1637           0 :         eat = min_t(int, len, skb_headlen(skb));
    1638           0 :         if (eat) {
    1639           0 :                 __skb_pull(skb, eat);
    1640           0 :                 len -= eat;
    1641           0 :                 if (!len)
    1642             :                         return 0;
    1643             :         }
    1644           0 :         eat = len;
    1645           0 :         k = 0;
    1646           0 :         shinfo = skb_shinfo(skb);
    1647           0 :         for (i = 0; i < shinfo->nr_frags; i++) {
    1648           0 :                 int size = skb_frag_size(&shinfo->frags[i]);
    1649             : 
    1650           0 :                 if (size <= eat) {
    1651           0 :                         skb_frag_unref(skb, i);
    1652           0 :                         eat -= size;
    1653             :                 } else {
    1654           0 :                         shinfo->frags[k] = shinfo->frags[i];
    1655           0 :                         if (eat) {
    1656           0 :                                 skb_frag_off_add(&shinfo->frags[k], eat);
    1657           0 :                                 skb_frag_size_sub(&shinfo->frags[k], eat);
    1658           0 :                                 eat = 0;
    1659             :                         }
    1660           0 :                         k++;
    1661             :                 }
    1662             :         }
    1663           0 :         shinfo->nr_frags = k;
    1664             : 
    1665           0 :         skb->data_len -= len;
    1666           0 :         skb->len = skb->data_len;
    1667           0 :         return len;
    1668             : }
    1669             : 
    1670             : /* Remove acked data from a packet in the transmit queue. */
    1671           0 : int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
    1672             : {
    1673           0 :         u32 delta_truesize;
    1674             : 
    1675           0 :         if (skb_unclone(skb, GFP_ATOMIC))
    1676             :                 return -ENOMEM;
    1677             : 
    1678           0 :         delta_truesize = __pskb_trim_head(skb, len);
    1679             : 
    1680           0 :         TCP_SKB_CB(skb)->seq += len;
    1681           0 :         skb->ip_summed = CHECKSUM_PARTIAL;
    1682             : 
    1683           0 :         if (delta_truesize) {
    1684           0 :                 skb->truesize           -= delta_truesize;
    1685           0 :                 sk_wmem_queued_add(sk, -delta_truesize);
    1686           0 :                 sk_mem_uncharge(sk, delta_truesize);
    1687             :         }
    1688             : 
    1689             :         /* Any change of skb->len requires recalculation of tso factor. */
    1690           0 :         if (tcp_skb_pcount(skb) > 1)
    1691           0 :                 tcp_set_skb_tso_segs(skb, tcp_skb_mss(skb));
    1692             : 
    1693             :         return 0;
    1694             : }
    1695             : 
    1696             : /* Calculate MSS not accounting any TCP options.  */
    1697           4 : static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
    1698             : {
    1699           4 :         const struct tcp_sock *tp = tcp_sk(sk);
    1700           4 :         const struct inet_connection_sock *icsk = inet_csk(sk);
    1701           4 :         int mss_now;
    1702             : 
    1703             :         /* Calculate base mss without TCP options:
    1704             :            It is MMS_S - sizeof(tcphdr) of rfc1122
    1705             :          */
    1706           4 :         mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
    1707             : 
    1708             :         /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
    1709           4 :         if (icsk->icsk_af_ops->net_frag_header_len) {
    1710           0 :                 const struct dst_entry *dst = __sk_dst_get(sk);
    1711             : 
    1712           0 :                 if (dst && dst_allfrag(dst))
    1713           0 :                         mss_now -= icsk->icsk_af_ops->net_frag_header_len;
    1714             :         }
    1715             : 
    1716             :         /* Clamp it (mss_clamp does not include tcp options) */
    1717           4 :         if (mss_now > tp->rx_opt.mss_clamp)
    1718             :                 mss_now = tp->rx_opt.mss_clamp;
    1719             : 
    1720             :         /* Now subtract optional transport overhead */
    1721           4 :         mss_now -= icsk->icsk_ext_hdr_len;
    1722             : 
    1723             :         /* Then reserve room for full set of TCP options and 8 bytes of data */
    1724           4 :         mss_now = max(mss_now, sock_net(sk)->ipv4.sysctl_tcp_min_snd_mss);
    1725           4 :         return mss_now;
    1726             : }
    1727             : 
    1728             : /* Calculate MSS. Not accounting for SACKs here.  */
    1729           4 : int tcp_mtu_to_mss(struct sock *sk, int pmtu)
    1730             : {
    1731             :         /* Subtract TCP options size, not including SACKs */
    1732           4 :         return __tcp_mtu_to_mss(sk, pmtu) -
    1733           4 :                (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
    1734             : }
    1735             : 
    1736             : /* Inverse of above */
    1737           4 : int tcp_mss_to_mtu(struct sock *sk, int mss)
    1738             : {
    1739           4 :         const struct tcp_sock *tp = tcp_sk(sk);
    1740           4 :         const struct inet_connection_sock *icsk = inet_csk(sk);
    1741           4 :         int mtu;
    1742             : 
    1743           4 :         mtu = mss +
    1744           4 :               tp->tcp_header_len +
    1745           4 :               icsk->icsk_ext_hdr_len +
    1746           4 :               icsk->icsk_af_ops->net_header_len;
    1747             : 
    1748             :         /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
    1749           4 :         if (icsk->icsk_af_ops->net_frag_header_len) {
    1750           0 :                 const struct dst_entry *dst = __sk_dst_get(sk);
    1751             : 
    1752           0 :                 if (dst && dst_allfrag(dst))
    1753           0 :                         mtu += icsk->icsk_af_ops->net_frag_header_len;
    1754             :         }
    1755           4 :         return mtu;
    1756             : }
    1757             : EXPORT_SYMBOL(tcp_mss_to_mtu);
    1758             : 
    1759             : /* MTU probing init per socket */
    1760           4 : void tcp_mtup_init(struct sock *sk)
    1761             : {
    1762           4 :         struct tcp_sock *tp = tcp_sk(sk);
    1763           4 :         struct inet_connection_sock *icsk = inet_csk(sk);
    1764           4 :         struct net *net = sock_net(sk);
    1765             : 
    1766           4 :         icsk->icsk_mtup.enabled = net->ipv4.sysctl_tcp_mtu_probing > 1;
    1767           4 :         icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
    1768           4 :                                icsk->icsk_af_ops->net_header_len;
    1769           4 :         icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, net->ipv4.sysctl_tcp_base_mss);
    1770           4 :         icsk->icsk_mtup.probe_size = 0;
    1771           4 :         if (icsk->icsk_mtup.enabled)
    1772           0 :                 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
    1773           4 : }
    1774             : EXPORT_SYMBOL(tcp_mtup_init);
    1775             : 
    1776             : /* This function synchronize snd mss to current pmtu/exthdr set.
    1777             : 
    1778             :    tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
    1779             :    for TCP options, but includes only bare TCP header.
    1780             : 
    1781             :    tp->rx_opt.mss_clamp is mss negotiated at connection setup.
    1782             :    It is minimum of user_mss and mss received with SYN.
    1783             :    It also does not include TCP options.
    1784             : 
    1785             :    inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
    1786             : 
    1787             :    tp->mss_cache is current effective sending mss, including
    1788             :    all tcp options except for SACKs. It is evaluated,
    1789             :    taking into account current pmtu, but never exceeds
    1790             :    tp->rx_opt.mss_clamp.
    1791             : 
    1792             :    NOTE1. rfc1122 clearly states that advertised MSS
    1793             :    DOES NOT include either tcp or ip options.
    1794             : 
    1795             :    NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
    1796             :    are READ ONLY outside this function.         --ANK (980731)
    1797             :  */
    1798           4 : unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
    1799             : {
    1800           4 :         struct tcp_sock *tp = tcp_sk(sk);
    1801           4 :         struct inet_connection_sock *icsk = inet_csk(sk);
    1802           4 :         int mss_now;
    1803             : 
    1804           4 :         if (icsk->icsk_mtup.search_high > pmtu)
    1805           0 :                 icsk->icsk_mtup.search_high = pmtu;
    1806             : 
    1807           4 :         mss_now = tcp_mtu_to_mss(sk, pmtu);
    1808           4 :         mss_now = tcp_bound_to_half_wnd(tp, mss_now);
    1809             : 
    1810             :         /* And store cached results */
    1811           4 :         icsk->icsk_pmtu_cookie = pmtu;
    1812           4 :         if (icsk->icsk_mtup.enabled)
    1813           0 :                 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
    1814           4 :         tp->mss_cache = mss_now;
    1815             : 
    1816           4 :         return mss_now;
    1817             : }
    1818             : EXPORT_SYMBOL(tcp_sync_mss);
    1819             : 
    1820             : /* Compute the current effective MSS, taking SACKs and IP options,
    1821             :  * and even PMTU discovery events into account.
    1822             :  */
    1823         496 : unsigned int tcp_current_mss(struct sock *sk)
    1824             : {
    1825         496 :         const struct tcp_sock *tp = tcp_sk(sk);
    1826         496 :         const struct dst_entry *dst = __sk_dst_get(sk);
    1827         496 :         u32 mss_now;
    1828         496 :         unsigned int header_len;
    1829         496 :         struct tcp_out_options opts;
    1830         496 :         struct tcp_md5sig_key *md5;
    1831             : 
    1832         496 :         mss_now = tp->mss_cache;
    1833             : 
    1834         496 :         if (dst) {
    1835         492 :                 u32 mtu = dst_mtu(dst);
    1836         492 :                 if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
    1837           0 :                         mss_now = tcp_sync_mss(sk, mtu);
    1838             :         }
    1839             : 
    1840         496 :         header_len = tcp_established_options(sk, NULL, &opts, &md5) +
    1841             :                      sizeof(struct tcphdr);
    1842             :         /* The mss_cache is sized based on tp->tcp_header_len, which assumes
    1843             :          * some common options. If this is an odd packet (because we have SACK
    1844             :          * blocks etc) then our calculated header_len will be different, and
    1845             :          * we have to adjust mss_now correspondingly */
    1846         496 :         if (header_len != tp->tcp_header_len) {
    1847           0 :                 int delta = (int) header_len - tp->tcp_header_len;
    1848           0 :                 mss_now -= delta;
    1849             :         }
    1850             : 
    1851         496 :         return mss_now;
    1852             : }
    1853             : 
    1854             : /* RFC2861, slow part. Adjust cwnd, after it was not full during one rto.
    1855             :  * As additional protections, we do not touch cwnd in retransmission phases,
    1856             :  * and if application hit its sndbuf limit recently.
    1857             :  */
    1858           0 : static void tcp_cwnd_application_limited(struct sock *sk)
    1859             : {
    1860           0 :         struct tcp_sock *tp = tcp_sk(sk);
    1861             : 
    1862           0 :         if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open &&
    1863           0 :             sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
    1864             :                 /* Limited by application or receiver window. */
    1865           0 :                 u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
    1866           0 :                 u32 win_used = max(tp->snd_cwnd_used, init_win);
    1867           0 :                 if (win_used < tp->snd_cwnd) {
    1868           0 :                         tp->snd_ssthresh = tcp_current_ssthresh(sk);
    1869           0 :                         tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1;
    1870             :                 }
    1871           0 :                 tp->snd_cwnd_used = 0;
    1872             :         }
    1873           0 :         tp->snd_cwnd_stamp = tcp_jiffies32;
    1874           0 : }
    1875             : 
    1876         364 : static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
    1877             : {
    1878         364 :         const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
    1879         364 :         struct tcp_sock *tp = tcp_sk(sk);
    1880             : 
    1881             :         /* Track the maximum number of outstanding packets in each
    1882             :          * window, and remember whether we were cwnd-limited then.
    1883             :          */
    1884         364 :         if (!before(tp->snd_una, tp->max_packets_seq) ||
    1885         126 :             tp->packets_out > tp->max_packets_out ||
    1886             :             is_cwnd_limited) {
    1887         343 :                 tp->max_packets_out = tp->packets_out;
    1888         343 :                 tp->max_packets_seq = tp->snd_nxt;
    1889         343 :                 tp->is_cwnd_limited = is_cwnd_limited;
    1890             :         }
    1891             : 
    1892         728 :         if (tcp_is_cwnd_limited(sk)) {
    1893             :                 /* Network is feed fully. */
    1894           0 :                 tp->snd_cwnd_used = 0;
    1895           0 :                 tp->snd_cwnd_stamp = tcp_jiffies32;
    1896             :         } else {
    1897             :                 /* Network starves. */
    1898         364 :                 if (tp->packets_out > tp->snd_cwnd_used)
    1899          11 :                         tp->snd_cwnd_used = tp->packets_out;
    1900             : 
    1901         364 :                 if (sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle &&
    1902         364 :                     (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto &&
    1903           0 :                     !ca_ops->cong_control)
    1904           0 :                         tcp_cwnd_application_limited(sk);
    1905             : 
    1906             :                 /* The following conditions together indicate the starvation
    1907             :                  * is caused by insufficient sender buffer:
    1908             :                  * 1) just sent some data (see tcp_write_xmit)
    1909             :                  * 2) not cwnd limited (this else condition)
    1910             :                  * 3) no more data to send (tcp_write_queue_empty())
    1911             :                  * 4) application is hitting buffer limit (SOCK_NOSPACE)
    1912             :                  */
    1913         364 :                 if (tcp_write_queue_empty(sk) && sk->sk_socket &&
    1914         364 :                     test_bit(SOCK_NOSPACE, &sk->sk_socket->flags) &&
    1915           0 :                     (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
    1916         364 :                         tcp_chrono_start(sk, TCP_CHRONO_SNDBUF_LIMITED);
    1917             :         }
    1918         364 : }
    1919             : 
    1920             : /* Minshall's variant of the Nagle send check. */
    1921           2 : static bool tcp_minshall_check(const struct tcp_sock *tp)
    1922             : {
    1923           2 :         return after(tp->snd_sml, tp->snd_una) &&
    1924           2 :                 !after(tp->snd_sml, tp->snd_nxt);
    1925             : }
    1926             : 
    1927             : /* Update snd_sml if this skb is under mss
    1928             :  * Note that a TSO packet might end with a sub-mss segment
    1929             :  * The test is really :
    1930             :  * if ((skb->len % mss) != 0)
    1931             :  *        tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
    1932             :  * But we can avoid doing the divide again given we already have
    1933             :  *  skb_pcount = skb->len / mss_now
    1934             :  */
    1935         364 : static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
    1936             :                                 const struct sk_buff *skb)
    1937             : {
    1938         364 :         if (skb->len < tcp_skb_pcount(skb) * mss_now)
    1939         364 :                 tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
    1940             : }
    1941             : 
    1942             : /* Return false, if packet can be sent now without violation Nagle's rules:
    1943             :  * 1. It is full sized. (provided by caller in %partial bool)
    1944             :  * 2. Or it contains FIN. (already checked by caller)
    1945             :  * 3. Or TCP_CORK is not set, and TCP_NODELAY is set.
    1946             :  * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
    1947             :  *    With Minshall's modification: all sent small packets are ACKed.
    1948             :  */
    1949         363 : static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
    1950             :                             int nonagle)
    1951             : {
    1952         363 :         return partial &&
    1953         363 :                 ((nonagle & TCP_NAGLE_CORK) ||
    1954          42 :                  (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
    1955             : }
    1956             : 
    1957             : /* Return how many segs we'd like on a TSO packet,
    1958             :  * to send one TSO packet per ms
    1959             :  */
    1960         422 : static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
    1961             :                             int min_tso_segs)
    1962             : {
    1963         422 :         u32 bytes, segs;
    1964             : 
    1965         422 :         bytes = min_t(unsigned long,
    1966             :                       sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift),
    1967             :                       sk->sk_gso_max_size - 1 - MAX_TCP_HEADER);
    1968             : 
    1969             :         /* Goal is to send at least one packet per ms,
    1970             :          * not one big TSO packet every 100 ms.
    1971             :          * This preserves ACK clocking and is consistent
    1972             :          * with tcp_tso_should_defer() heuristic.
    1973             :          */
    1974         422 :         segs = max_t(u32, bytes / mss_now, min_tso_segs);
    1975             : 
    1976         422 :         return segs;
    1977             : }
    1978             : 
    1979             : /* Return the number of segments we want in the skb we are transmitting.
    1980             :  * See if congestion control module wants to decide; otherwise, autosize.
    1981             :  */
    1982         422 : static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)
    1983             : {
    1984         422 :         const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
    1985         422 :         u32 min_tso, tso_segs;
    1986             : 
    1987         844 :         min_tso = ca_ops->min_tso_segs ?
    1988         422 :                         ca_ops->min_tso_segs(sk) :
    1989         422 :                         sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs;
    1990             : 
    1991         422 :         tso_segs = tcp_tso_autosize(sk, mss_now, min_tso);
    1992         422 :         return min_t(u32, tso_segs, sk->sk_gso_max_segs);
    1993             : }
    1994             : 
    1995             : /* Returns the portion of skb which can be sent right away */
    1996           0 : static unsigned int tcp_mss_split_point(const struct sock *sk,
    1997             :                                         const struct sk_buff *skb,
    1998             :                                         unsigned int mss_now,
    1999             :                                         unsigned int max_segs,
    2000             :                                         int nonagle)
    2001             : {
    2002           0 :         const struct tcp_sock *tp = tcp_sk(sk);
    2003           0 :         u32 partial, needed, window, max_len;
    2004             : 
    2005           0 :         window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
    2006           0 :         max_len = mss_now * max_segs;
    2007             : 
    2008           0 :         if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
    2009             :                 return max_len;
    2010             : 
    2011           0 :         needed = min(skb->len, window);
    2012             : 
    2013           0 :         if (max_len <= needed)
    2014             :                 return max_len;
    2015             : 
    2016           0 :         partial = needed % mss_now;
    2017             :         /* If last segment is not a full MSS, check if Nagle rules allow us
    2018             :          * to include this last segment in this skb.
    2019             :          * Otherwise, we'll split the skb at last MSS boundary
    2020             :          */
    2021           0 :         if (tcp_nagle_check(partial != 0, tp, nonagle))
    2022           0 :                 return needed - partial;
    2023             : 
    2024             :         return needed;
    2025             : }
    2026             : 
    2027             : /* Can at least one segment of SKB be sent right now, according to the
    2028             :  * congestion window rules?  If so, return how many segments are allowed.
    2029             :  */
    2030         366 : static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp,
    2031             :                                          const struct sk_buff *skb)
    2032             : {
    2033         366 :         u32 in_flight, cwnd, halfcwnd;
    2034             : 
    2035             :         /* Don't be strict about the congestion window for the final FIN.  */
    2036         366 :         if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) &&
    2037           3 :             tcp_skb_pcount(skb) == 1)
    2038             :                 return 1;
    2039             : 
    2040         363 :         in_flight = tcp_packets_in_flight(tp);
    2041         363 :         cwnd = tp->snd_cwnd;
    2042         363 :         if (in_flight >= cwnd)
    2043             :                 return 0;
    2044             : 
    2045             :         /* For better scheduling, ensure we have at least
    2046             :          * 2 GSO packets in flight.
    2047             :          */
    2048         363 :         halfcwnd = max(cwnd >> 1, 1U);
    2049         363 :         return min(halfcwnd, cwnd - in_flight);
    2050             : }
    2051             : 
    2052             : /* Initialize TSO state of a skb.
    2053             :  * This must be invoked the first time we consider transmitting
    2054             :  * SKB onto the wire.
    2055             :  */
    2056         366 : static int tcp_init_tso_segs(struct sk_buff *skb, unsigned int mss_now)
    2057             : {
    2058         366 :         int tso_segs = tcp_skb_pcount(skb);
    2059             : 
    2060         366 :         if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
    2061         361 :                 tcp_set_skb_tso_segs(skb, mss_now);
    2062         361 :                 tso_segs = tcp_skb_pcount(skb);
    2063             :         }
    2064         366 :         return tso_segs;
    2065             : }
    2066             : 
    2067             : 
    2068             : /* Return true if the Nagle test allows this packet to be
    2069             :  * sent now.
    2070             :  */
    2071         366 : static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
    2072             :                                   unsigned int cur_mss, int nonagle)
    2073             : {
    2074             :         /* Nagle rule does not apply to frames, which sit in the middle of the
    2075             :          * write_queue (they have no chances to get new data).
    2076             :          *
    2077             :          * This is implemented in the callers, where they modify the 'nonagle'
    2078             :          * argument based upon the location of SKB in the send queue.
    2079             :          */
    2080         366 :         if (nonagle & TCP_NAGLE_PUSH)
    2081             :                 return true;
    2082             : 
    2083             :         /* Don't use the nagle rule for urgent data (or for the final FIN). */
    2084         366 :         if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
    2085             :                 return true;
    2086             : 
    2087         363 :         if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))
    2088         361 :                 return true;
    2089             : 
    2090             :         return false;
    2091             : }
    2092             : 
    2093             : /* Does at least the first segment of SKB fit into the send window? */
    2094         366 : static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
    2095             :                              const struct sk_buff *skb,
    2096             :                              unsigned int cur_mss)
    2097             : {
    2098         366 :         u32 end_seq = TCP_SKB_CB(skb)->end_seq;
    2099             : 
    2100         366 :         if (skb->len > cur_mss)
    2101           0 :                 end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
    2102             : 
    2103         366 :         return !after(end_seq, tcp_wnd_end(tp));
    2104             : }
    2105             : 
    2106             : /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
    2107             :  * which is put after SKB on the list.  It is very much like
    2108             :  * tcp_fragment() except that it may make several kinds of assumptions
    2109             :  * in order to speed up the splitting operation.  In particular, we
    2110             :  * know that all the data is in scatter-gather pages, and that the
    2111             :  * packet has never been sent out before (and thus is not cloned).
    2112             :  */
    2113           0 : static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
    2114             :                         unsigned int mss_now, gfp_t gfp)
    2115             : {
    2116           0 :         int nlen = skb->len - len;
    2117           0 :         struct sk_buff *buff;
    2118           0 :         u8 flags;
    2119             : 
    2120             :         /* All of a TSO frame must be composed of paged data.  */
    2121           0 :         if (skb->len != skb->data_len)
    2122           0 :                 return tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
    2123             :                                     skb, len, mss_now, gfp);
    2124             : 
    2125           0 :         buff = sk_stream_alloc_skb(sk, 0, gfp, true);
    2126           0 :         if (unlikely(!buff))
    2127             :                 return -ENOMEM;
    2128           0 :         skb_copy_decrypted(buff, skb);
    2129           0 :         mptcp_skb_ext_copy(buff, skb);
    2130             : 
    2131           0 :         sk_wmem_queued_add(sk, buff->truesize);
    2132           0 :         sk_mem_charge(sk, buff->truesize);
    2133           0 :         buff->truesize += nlen;
    2134           0 :         skb->truesize -= nlen;
    2135             : 
    2136             :         /* Correct the sequence numbers. */
    2137           0 :         TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
    2138           0 :         TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
    2139           0 :         TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
    2140             : 
    2141             :         /* PSH and FIN should only be set in the second packet. */
    2142           0 :         flags = TCP_SKB_CB(skb)->tcp_flags;
    2143           0 :         TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
    2144           0 :         TCP_SKB_CB(buff)->tcp_flags = flags;
    2145             : 
    2146             :         /* This packet was never sent out yet, so no SACK bits. */
    2147           0 :         TCP_SKB_CB(buff)->sacked = 0;
    2148             : 
    2149           0 :         tcp_skb_fragment_eor(skb, buff);
    2150             : 
    2151           0 :         buff->ip_summed = CHECKSUM_PARTIAL;
    2152           0 :         skb_split(skb, buff, len);
    2153           0 :         tcp_fragment_tstamp(skb, buff);
    2154             : 
    2155             :         /* Fix up tso_factor for both original and new SKB.  */
    2156           0 :         tcp_set_skb_tso_segs(skb, mss_now);
    2157           0 :         tcp_set_skb_tso_segs(buff, mss_now);
    2158             : 
    2159             :         /* Link BUFF into the send queue. */
    2160           0 :         __skb_header_release(buff);
    2161           0 :         tcp_insert_write_queue_after(skb, buff, sk, TCP_FRAG_IN_WRITE_QUEUE);
    2162             : 
    2163           0 :         return 0;
    2164             : }
    2165             : 
    2166             : /* Try to defer sending, if possible, in order to minimize the amount
    2167             :  * of TSO splitting we do.  View it as a kind of TSO Nagle test.
    2168             :  *
    2169             :  * This algorithm is from John Heffner.
    2170             :  */
    2171           0 : static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
    2172             :                                  bool *is_cwnd_limited,
    2173             :                                  bool *is_rwnd_limited,
    2174             :                                  u32 max_segs)
    2175             : {
    2176           0 :         const struct inet_connection_sock *icsk = inet_csk(sk);
    2177           0 :         u32 send_win, cong_win, limit, in_flight;
    2178           0 :         struct tcp_sock *tp = tcp_sk(sk);
    2179           0 :         struct sk_buff *head;
    2180           0 :         int win_divisor;
    2181           0 :         s64 delta;
    2182             : 
    2183           0 :         if (icsk->icsk_ca_state >= TCP_CA_Recovery)
    2184           0 :                 goto send_now;
    2185             : 
    2186             :         /* Avoid bursty behavior by allowing defer
    2187             :          * only if the last write was recent (1 ms).
    2188             :          * Note that tp->tcp_wstamp_ns can be in the future if we have
    2189             :          * packets waiting in a qdisc or device for EDT delivery.
    2190             :          */
    2191           0 :         delta = tp->tcp_clock_cache - tp->tcp_wstamp_ns - NSEC_PER_MSEC;
    2192           0 :         if (delta > 0)
    2193           0 :                 goto send_now;
    2194             : 
    2195           0 :         in_flight = tcp_packets_in_flight(tp);
    2196             : 
    2197           0 :         BUG_ON(tcp_skb_pcount(skb) <= 1);
    2198           0 :         BUG_ON(tp->snd_cwnd <= in_flight);
    2199             : 
    2200           0 :         send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
    2201             : 
    2202             :         /* From in_flight test above, we know that cwnd > in_flight.  */
    2203           0 :         cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;
    2204             : 
    2205           0 :         limit = min(send_win, cong_win);
    2206             : 
    2207             :         /* If a full-sized TSO skb can be sent, do it. */
    2208           0 :         if (limit >= max_segs * tp->mss_cache)
    2209           0 :                 goto send_now;
    2210             : 
    2211             :         /* Middle in queue won't get any more data, full sendable already? */
    2212           0 :         if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
    2213           0 :                 goto send_now;
    2214             : 
    2215           0 :         win_divisor = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_win_divisor);
    2216           0 :         if (win_divisor) {
    2217           0 :                 u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);
    2218             : 
    2219             :                 /* If at least some fraction of a window is available,
    2220             :                  * just use it.
    2221             :                  */
    2222           0 :                 chunk /= win_divisor;
    2223           0 :                 if (limit >= chunk)
    2224           0 :                         goto send_now;
    2225             :         } else {
    2226             :                 /* Different approach, try not to defer past a single
    2227             :                  * ACK.  Receiver should ACK every other full sized
    2228             :                  * frame, so if we have space for more than 3 frames
    2229             :                  * then send now.
    2230             :                  */
    2231           0 :                 if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
    2232           0 :                         goto send_now;
    2233             :         }
    2234             : 
    2235             :         /* TODO : use tsorted_sent_queue ? */
    2236           0 :         head = tcp_rtx_queue_head(sk);
    2237           0 :         if (!head)
    2238           0 :                 goto send_now;
    2239           0 :         delta = tp->tcp_clock_cache - head->tstamp;
    2240             :         /* If next ACK is likely to come too late (half srtt), do not defer */
    2241           0 :         if ((s64)(delta - (u64)NSEC_PER_USEC * (tp->srtt_us >> 4)) < 0)
    2242           0 :                 goto send_now;
    2243             : 
    2244             :         /* Ok, it looks like it is advisable to defer.
    2245             :          * Three cases are tracked :
    2246             :          * 1) We are cwnd-limited
    2247             :          * 2) We are rwnd-limited
    2248             :          * 3) We are application limited.
    2249             :          */
    2250           0 :         if (cong_win < send_win) {
    2251           0 :                 if (cong_win <= skb->len) {
    2252           0 :                         *is_cwnd_limited = true;
    2253           0 :                         return true;
    2254             :                 }
    2255             :         } else {
    2256           0 :                 if (send_win <= skb->len) {
    2257           0 :                         *is_rwnd_limited = true;
    2258           0 :                         return true;
    2259             :                 }
    2260             :         }
    2261             : 
    2262             :         /* If this packet won't get more data, do not wait. */
    2263           0 :         if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) ||
    2264             :             TCP_SKB_CB(skb)->eor)
    2265           0 :                 goto send_now;
    2266             : 
    2267             :         return true;
    2268             : 
    2269             : send_now:
    2270             :         return false;
    2271             : }
    2272             : 
    2273           0 : static inline void tcp_mtu_check_reprobe(struct sock *sk)
    2274             : {
    2275           0 :         struct inet_connection_sock *icsk = inet_csk(sk);
    2276           0 :         struct tcp_sock *tp = tcp_sk(sk);
    2277           0 :         struct net *net = sock_net(sk);
    2278           0 :         u32 interval;
    2279           0 :         s32 delta;
    2280             : 
    2281           0 :         interval = net->ipv4.sysctl_tcp_probe_interval;
    2282           0 :         delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp;
    2283           0 :         if (unlikely(delta >= interval * HZ)) {
    2284           0 :                 int mss = tcp_current_mss(sk);
    2285             : 
    2286             :                 /* Update current search range */
    2287           0 :                 icsk->icsk_mtup.probe_size = 0;
    2288           0 :                 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp +
    2289           0 :                         sizeof(struct tcphdr) +
    2290           0 :                         icsk->icsk_af_ops->net_header_len;
    2291           0 :                 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss);
    2292             : 
    2293             :                 /* Update probe time stamp */
    2294           0 :                 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
    2295             :         }
    2296           0 : }
    2297             : 
    2298           0 : static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len)
    2299             : {
    2300           0 :         struct sk_buff *skb, *next;
    2301             : 
    2302           0 :         skb = tcp_send_head(sk);
    2303           0 :         tcp_for_write_queue_from_safe(skb, next, sk) {
    2304           0 :                 if (len <= skb->len)
    2305             :                         break;
    2306             : 
    2307           0 :                 if (unlikely(TCP_SKB_CB(skb)->eor) || tcp_has_tx_tstamp(skb))
    2308             :                         return false;
    2309             : 
    2310           0 :                 len -= skb->len;
    2311             :         }
    2312             : 
    2313             :         return true;
    2314             : }
    2315             : 
    2316             : /* Create a new MTU probe if we are ready.
    2317             :  * MTU probe is regularly attempting to increase the path MTU by
    2318             :  * deliberately sending larger packets.  This discovers routing
    2319             :  * changes resulting in larger path MTUs.
    2320             :  *
    2321             :  * Returns 0 if we should wait to probe (no cwnd available),
    2322             :  *         1 if a probe was sent,
    2323             :  *         -1 otherwise
    2324             :  */
    2325         422 : static int tcp_mtu_probe(struct sock *sk)
    2326             : {
    2327         422 :         struct inet_connection_sock *icsk = inet_csk(sk);
    2328         422 :         struct tcp_sock *tp = tcp_sk(sk);
    2329         422 :         struct sk_buff *skb, *nskb, *next;
    2330         422 :         struct net *net = sock_net(sk);
    2331         422 :         int probe_size;
    2332         422 :         int size_needed;
    2333         422 :         int copy, len;
    2334         422 :         int mss_now;
    2335         422 :         int interval;
    2336             : 
    2337             :         /* Not currently probing/verifying,
    2338             :          * not in recovery,
    2339             :          * have enough cwnd, and
    2340             :          * not SACKing (the variable headers throw things off)
    2341             :          */
    2342         422 :         if (likely(!icsk->icsk_mtup.enabled ||
    2343             :                    icsk->icsk_mtup.probe_size ||
    2344             :                    inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
    2345             :                    tp->snd_cwnd < 11 ||
    2346             :                    tp->rx_opt.num_sacks || tp->rx_opt.dsack))
    2347             :                 return -1;
    2348             : 
    2349             :         /* Use binary search for probe_size between tcp_mss_base,
    2350             :          * and current mss_clamp. if (search_high - search_low)
    2351             :          * smaller than a threshold, backoff from probing.
    2352             :          */
    2353           0 :         mss_now = tcp_current_mss(sk);
    2354           0 :         probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high +
    2355           0 :                                     icsk->icsk_mtup.search_low) >> 1);
    2356           0 :         size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
    2357           0 :         interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low;
    2358             :         /* When misfortune happens, we are reprobing actively,
    2359             :          * and then reprobe timer has expired. We stick with current
    2360             :          * probing process by not resetting search range to its orignal.
    2361             :          */
    2362           0 :         if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) ||
    2363           0 :                 interval < net->ipv4.sysctl_tcp_probe_threshold) {
    2364             :                 /* Check whether enough time has elaplased for
    2365             :                  * another round of probing.
    2366             :                  */
    2367           0 :                 tcp_mtu_check_reprobe(sk);
    2368           0 :                 return -1;
    2369             :         }
    2370             : 
    2371             :         /* Have enough data in the send queue to probe? */
    2372           0 :         if (tp->write_seq - tp->snd_nxt < size_needed)
    2373             :                 return -1;
    2374             : 
    2375           0 :         if (tp->snd_wnd < size_needed)
    2376             :                 return -1;
    2377           0 :         if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
    2378             :                 return 0;
    2379             : 
    2380             :         /* Do we need to wait to drain cwnd? With none in flight, don't stall */
    2381           0 :         if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) {
    2382           0 :                 if (!tcp_packets_in_flight(tp))
    2383             :                         return -1;
    2384             :                 else
    2385           0 :                         return 0;
    2386             :         }
    2387             : 
    2388           0 :         if (!tcp_can_coalesce_send_queue_head(sk, probe_size))
    2389             :                 return -1;
    2390             : 
    2391             :         /* We're allowed to probe.  Build it now. */
    2392           0 :         nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC, false);
    2393           0 :         if (!nskb)
    2394             :                 return -1;
    2395           0 :         sk_wmem_queued_add(sk, nskb->truesize);
    2396           0 :         sk_mem_charge(sk, nskb->truesize);
    2397             : 
    2398           0 :         skb = tcp_send_head(sk);
    2399           0 :         skb_copy_decrypted(nskb, skb);
    2400           0 :         mptcp_skb_ext_copy(nskb, skb);
    2401             : 
    2402           0 :         TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
    2403           0 :         TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
    2404           0 :         TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
    2405           0 :         TCP_SKB_CB(nskb)->sacked = 0;
    2406           0 :         nskb->csum = 0;
    2407           0 :         nskb->ip_summed = CHECKSUM_PARTIAL;
    2408             : 
    2409           0 :         tcp_insert_write_queue_before(nskb, skb, sk);
    2410           0 :         tcp_highest_sack_replace(sk, skb, nskb);
    2411             : 
    2412           0 :         len = 0;
    2413           0 :         tcp_for_write_queue_from_safe(skb, next, sk) {
    2414           0 :                 copy = min_t(int, skb->len, probe_size - len);
    2415           0 :                 skb_copy_bits(skb, 0, skb_put(nskb, copy), copy);
    2416             : 
    2417           0 :                 if (skb->len <= copy) {
    2418             :                         /* We've eaten all the data from this skb.
    2419             :                          * Throw it away. */
    2420           0 :                         TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
    2421             :                         /* If this is the last SKB we copy and eor is set
    2422             :                          * we need to propagate it to the new skb.
    2423             :                          */
    2424           0 :                         TCP_SKB_CB(nskb)->eor = TCP_SKB_CB(skb)->eor;
    2425           0 :                         tcp_skb_collapse_tstamp(nskb, skb);
    2426           0 :                         tcp_unlink_write_queue(skb, sk);
    2427           0 :                         sk_wmem_free_skb(sk, skb);
    2428             :                 } else {
    2429           0 :                         TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
    2430             :                                                    ~(TCPHDR_FIN|TCPHDR_PSH);
    2431           0 :                         if (!skb_shinfo(skb)->nr_frags) {
    2432           0 :                                 skb_pull(skb, copy);
    2433             :                         } else {
    2434           0 :                                 __pskb_trim_head(skb, copy);
    2435           0 :                                 tcp_set_skb_tso_segs(skb, mss_now);
    2436             :                         }
    2437           0 :                         TCP_SKB_CB(skb)->seq += copy;
    2438             :                 }
    2439             : 
    2440           0 :                 len += copy;
    2441             : 
    2442           0 :                 if (len >= probe_size)
    2443             :                         break;
    2444             :         }
    2445           0 :         tcp_init_tso_segs(nskb, nskb->len);
    2446             : 
    2447             :         /* We're ready to send.  If this fails, the probe will
    2448             :          * be resegmented into mss-sized pieces by tcp_write_xmit().
    2449             :          */
    2450           0 :         if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
    2451             :                 /* Decrement cwnd here because we are sending
    2452             :                  * effectively two packets. */
    2453           0 :                 tp->snd_cwnd--;
    2454           0 :                 tcp_event_new_data_sent(sk, nskb);
    2455             : 
    2456           0 :                 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
    2457           0 :                 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
    2458           0 :                 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
    2459             : 
    2460           0 :                 return 1;
    2461             :         }
    2462             : 
    2463             :         return -1;
    2464             : }
    2465             : 
    2466         366 : static bool tcp_pacing_check(struct sock *sk)
    2467             : {
    2468         366 :         struct tcp_sock *tp = tcp_sk(sk);
    2469             : 
    2470         366 :         if (!tcp_needs_internal_pacing(sk))
    2471             :                 return false;
    2472             : 
    2473           0 :         if (tp->tcp_wstamp_ns <= tp->tcp_clock_cache)
    2474             :                 return false;
    2475             : 
    2476           0 :         if (!hrtimer_is_queued(&tp->pacing_timer)) {
    2477           0 :                 hrtimer_start(&tp->pacing_timer,
    2478             :                               ns_to_ktime(tp->tcp_wstamp_ns),
    2479             :                               HRTIMER_MODE_ABS_PINNED_SOFT);
    2480           0 :                 sock_hold(sk);
    2481             :         }
    2482             :         return true;
    2483             : }
    2484             : 
    2485             : /* TCP Small Queues :
    2486             :  * Control number of packets in qdisc/devices to two packets / or ~1 ms.
    2487             :  * (These limits are doubled for retransmits)
    2488             :  * This allows for :
    2489             :  *  - better RTT estimation and ACK scheduling
    2490             :  *  - faster recovery
    2491             :  *  - high rates
    2492             :  * Alas, some drivers / subsystems require a fair amount
    2493             :  * of queued bytes to ensure line rate.
    2494             :  * One example is wifi aggregation (802.11 AMPDU)
    2495             :  */
    2496         364 : static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
    2497             :                                   unsigned int factor)
    2498             : {
    2499         364 :         unsigned long limit;
    2500             : 
    2501         364 :         limit = max_t(unsigned long,
    2502             :                       2 * skb->truesize,
    2503             :                       sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift));
    2504         364 :         if (sk->sk_pacing_status == SK_PACING_NONE)
    2505         364 :                 limit = min_t(unsigned long, limit,
    2506             :                               sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes);
    2507         364 :         limit <<= factor;
    2508             : 
    2509         364 :         if (static_branch_unlikely(&tcp_tx_delay_enabled) &&
    2510           0 :             tcp_sk(sk)->tcp_tx_delay) {
    2511           0 :                 u64 extra_bytes = (u64)sk->sk_pacing_rate * tcp_sk(sk)->tcp_tx_delay;
    2512             : 
    2513             :                 /* TSQ is based on skb truesize sum (sk_wmem_alloc), so we
    2514             :                  * approximate our needs assuming an ~100% skb->truesize overhead.
    2515             :                  * USEC_PER_SEC is approximated by 2^20.
    2516             :                  * do_div(extra_bytes, USEC_PER_SEC/2) is replaced by a right shift.
    2517             :                  */
    2518           0 :                 extra_bytes >>= (20 - 1);
    2519           0 :                 limit += extra_bytes;
    2520             :         }
    2521         364 :         if (refcount_read(&sk->sk_wmem_alloc) > limit) {
    2522             :                 /* Always send skb if rtx queue is empty.
    2523             :                  * No need to wait for TX completion to call us back,
    2524             :                  * after softirq/tasklet schedule.
    2525             :                  * This helps when TX completions are delayed too much.
    2526             :                  */
    2527           0 :                 if (tcp_rtx_queue_empty(sk))
    2528             :                         return false;
    2529             : 
    2530           0 :                 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
    2531             :                 /* It is possible TX completion already happened
    2532             :                  * before we set TSQ_THROTTLED, so we must
    2533             :                  * test again the condition.
    2534             :                  */
    2535           0 :                 smp_mb__after_atomic();
    2536           0 :                 if (refcount_read(&sk->sk_wmem_alloc) > limit)
    2537           0 :                         return true;
    2538             :         }
    2539             :         return false;
    2540             : }
    2541             : 
    2542         478 : static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)
    2543             : {
    2544         478 :         const u32 now = tcp_jiffies32;
    2545         478 :         enum tcp_chrono old = tp->chrono_type;
    2546             : 
    2547         478 :         if (old > TCP_CHRONO_UNSPEC)
    2548         234 :                 tp->chrono_stat[old - 1] += now - tp->chrono_start;
    2549         478 :         tp->chrono_start = now;
    2550         478 :         tp->chrono_type = new;
    2551         478 : }
    2552             : 
    2553         364 : void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
    2554             : {
    2555         364 :         struct tcp_sock *tp = tcp_sk(sk);
    2556             : 
    2557             :         /* If there are multiple conditions worthy of tracking in a
    2558             :          * chronograph then the highest priority enum takes precedence
    2559             :          * over the other conditions. So that if something "more interesting"
    2560             :          * starts happening, stop the previous chrono and start a new one.
    2561             :          */
    2562         364 :         if (type > tp->chrono_type)
    2563         220 :                 tcp_chrono_set(tp, type);
    2564         361 : }
    2565             : 
    2566         660 : void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type)
    2567             : {
    2568         660 :         struct tcp_sock *tp = tcp_sk(sk);
    2569             : 
    2570             : 
    2571             :         /* There are multiple conditions worthy of tracking in a
    2572             :          * chronograph, so that the highest priority enum takes
    2573             :          * precedence over the other conditions (see tcp_chrono_start).
    2574             :          * If a condition stops, we only stop chrono tracking if
    2575             :          * it's the "most interesting" or current chrono we are
    2576             :          * tracking and starts busy chrono if we have pending data.
    2577             :          */
    2578         918 :         if (tcp_rtx_and_write_queues_empty(sk))
    2579         244 :                 tcp_chrono_set(tp, TCP_CHRONO_UNSPEC);
    2580         416 :         else if (type == tp->chrono_type)
    2581          14 :                 tcp_chrono_set(tp, TCP_CHRONO_BUSY);
    2582         660 : }
    2583             : 
    2584             : /* This routine writes packets to the network.  It advances the
    2585             :  * send_head.  This happens as incoming acks open up the remote
    2586             :  * window for us.
    2587             :  *
    2588             :  * LARGESEND note: !tcp_urg_mode is overkill, only frames between
    2589             :  * snd_up-64k-mss .. snd_up cannot be large. However, taking into
    2590             :  * account rare use of URG, this is not a big flaw.
    2591             :  *
    2592             :  * Send at most one packet when push_one > 0. Temporarily ignore
    2593             :  * cwnd limit to force at most one packet out when push_one == 2.
    2594             : 
    2595             :  * Returns true, if no segments are in flight and we have queued segments,
    2596             :  * but cannot send anything now because of SWS or another problem.
    2597             :  */
    2598         422 : static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
    2599             :                            int push_one, gfp_t gfp)
    2600             : {
    2601         422 :         struct tcp_sock *tp = tcp_sk(sk);
    2602         422 :         struct sk_buff *skb;
    2603         422 :         unsigned int tso_segs, sent_pkts;
    2604         422 :         int cwnd_quota;
    2605         422 :         int result;
    2606         422 :         bool is_cwnd_limited = false, is_rwnd_limited = false;
    2607         422 :         u32 max_segs;
    2608             : 
    2609         422 :         sent_pkts = 0;
    2610             : 
    2611         422 :         tcp_mstamp_refresh(tp);
    2612         422 :         if (!push_one) {
    2613             :                 /* Do MTU probing. */
    2614         422 :                 result = tcp_mtu_probe(sk);
    2615         422 :                 if (!result) {
    2616             :                         return false;
    2617         422 :                 } else if (result > 0) {
    2618           0 :                         sent_pkts = 1;
    2619             :                 }
    2620             :         }
    2621             : 
    2622         422 :         max_segs = tcp_tso_segs(sk, mss_now);
    2623         786 :         while ((skb = tcp_send_head(sk))) {
    2624         366 :                 unsigned int limit;
    2625             : 
    2626         366 :                 if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) {
    2627             :                         /* "skb_mstamp_ns" is used as a start point for the retransmit timer */
    2628           0 :                         skb->skb_mstamp_ns = tp->tcp_wstamp_ns = tp->tcp_clock_cache;
    2629           0 :                         list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
    2630           0 :                         tcp_init_tso_segs(skb, mss_now);
    2631           0 :                         goto repair; /* Skip network transmission */
    2632             :                 }
    2633             : 
    2634         366 :                 if (tcp_pacing_check(sk))
    2635             :                         break;
    2636             : 
    2637         366 :                 tso_segs = tcp_init_tso_segs(skb, mss_now);
    2638         366 :                 BUG_ON(!tso_segs);
    2639             : 
    2640         366 :                 cwnd_quota = tcp_cwnd_test(tp, skb);
    2641         366 :                 if (!cwnd_quota) {
    2642           0 :                         if (push_one == 2)
    2643             :                                 /* Force out a loss probe pkt. */
    2644             :                                 cwnd_quota = 1;
    2645             :                         else
    2646             :                                 break;
    2647             :                 }
    2648             : 
    2649         366 :                 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) {
    2650           0 :                         is_rwnd_limited = true;
    2651           0 :                         break;
    2652             :                 }
    2653             : 
    2654         366 :                 if (tso_segs == 1) {
    2655         366 :                         if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
    2656             :                                                      (tcp_skb_is_last(sk, skb) ?
    2657             :                                                       nonagle : TCP_NAGLE_PUSH))))
    2658             :                                 break;
    2659             :                 } else {
    2660           0 :                         if (!push_one &&
    2661           0 :                             tcp_tso_should_defer(sk, skb, &is_cwnd_limited,
    2662             :                                                  &is_rwnd_limited, max_segs))
    2663             :                                 break;
    2664             :                 }
    2665             : 
    2666         364 :                 limit = mss_now;
    2667         364 :                 if (tso_segs > 1 && !tcp_urg_mode(tp))
    2668           0 :                         limit = tcp_mss_split_point(sk, skb, mss_now,
    2669           0 :                                                     min_t(unsigned int,
    2670             :                                                           cwnd_quota,
    2671             :                                                           max_segs),
    2672             :                                                     nonagle);
    2673             : 
    2674         364 :                 if (skb->len > limit &&
    2675           0 :                     unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
    2676             :                         break;
    2677             : 
    2678         364 :                 if (tcp_small_queue_check(sk, skb, 0))
    2679             :                         break;
    2680             : 
    2681             :                 /* Argh, we hit an empty skb(), presumably a thread
    2682             :                  * is sleeping in sendmsg()/sk_stream_wait_memory().
    2683             :                  * We do not want to send a pure-ack packet and have
    2684             :                  * a strange looking rtx queue with empty packet(s).
    2685             :                  */
    2686         364 :                 if (TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq)
    2687             :                         break;
    2688             : 
    2689         364 :                 if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
    2690             :                         break;
    2691             : 
    2692         364 : repair:
    2693             :                 /* Advance the send_head.  This one is sent out.
    2694             :                  * This call will increment packets_out.
    2695             :                  */
    2696         364 :                 tcp_event_new_data_sent(sk, skb);
    2697             : 
    2698         364 :                 tcp_minshall_update(tp, mss_now, skb);
    2699         364 :                 sent_pkts += tcp_skb_pcount(skb);
    2700             : 
    2701         364 :                 if (push_one)
    2702             :                         break;
    2703             :         }
    2704             : 
    2705         422 :         if (is_rwnd_limited)
    2706           0 :                 tcp_chrono_start(sk, TCP_CHRONO_RWND_LIMITED);
    2707             :         else
    2708         422 :                 tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED);
    2709             : 
    2710         422 :         is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tp->snd_cwnd);
    2711         422 :         if (likely(sent_pkts || is_cwnd_limited))
    2712         364 :                 tcp_cwnd_validate(sk, is_cwnd_limited);
    2713             : 
    2714         422 :         if (likely(sent_pkts)) {
    2715         364 :                 if (tcp_in_cwnd_reduction(sk))
    2716           0 :                         tp->prr_out += sent_pkts;
    2717             : 
    2718             :                 /* Send one loss probe per tail loss episode. */
    2719         364 :                 if (push_one != 2)
    2720         364 :                         tcp_schedule_loss_probe(sk, false);
    2721         364 :                 return false;
    2722             :         }
    2723          78 :         return !tp->packets_out && !tcp_write_queue_empty(sk);
    2724             : }
    2725             : 
    2726         715 : bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto)
    2727             : {
    2728         715 :         struct inet_connection_sock *icsk = inet_csk(sk);
    2729         715 :         struct tcp_sock *tp = tcp_sk(sk);
    2730         715 :         u32 timeout, rto_delta_us;
    2731         715 :         int early_retrans;
    2732             : 
    2733             :         /* Don't do any loss probe on a Fast Open connection before 3WHS
    2734             :          * finishes.
    2735             :          */
    2736         715 :         if (rcu_access_pointer(tp->fastopen_rsk))
    2737             :                 return false;
    2738             : 
    2739         715 :         early_retrans = sock_net(sk)->ipv4.sysctl_tcp_early_retrans;
    2740             :         /* Schedule a loss probe in 2*RTT for SACK capable connections
    2741             :          * not in loss recovery, that are either limited by cwnd or application.
    2742             :          */
    2743         715 :         if ((early_retrans != 3 && early_retrans != 4) ||
    2744         715 :             !tp->packets_out || !tcp_is_sack(tp) ||
    2745           0 :             (icsk->icsk_ca_state != TCP_CA_Open &&
    2746             :              icsk->icsk_ca_state != TCP_CA_CWR))
    2747             :                 return false;
    2748             : 
    2749             :         /* Probe timeout is 2*rtt. Add minimum RTO to account
    2750             :          * for delayed ack when there's one outstanding packet. If no RTT
    2751             :          * sample is available then probe after TCP_TIMEOUT_INIT.
    2752             :          */
    2753           0 :         if (tp->srtt_us) {
    2754           0 :                 timeout = usecs_to_jiffies(tp->srtt_us >> 2);
    2755           0 :                 if (tp->packets_out == 1)
    2756           0 :                         timeout += TCP_RTO_MIN;
    2757             :                 else
    2758           0 :                         timeout += TCP_TIMEOUT_MIN;
    2759             :         } else {
    2760             :                 timeout = TCP_TIMEOUT_INIT;
    2761             :         }
    2762             : 
    2763             :         /* If the RTO formula yields an earlier time, then use that time. */
    2764           0 :         rto_delta_us = advancing_rto ?
    2765           0 :                         jiffies_to_usecs(inet_csk(sk)->icsk_rto) :
    2766           0 :                         tcp_rto_delta_us(sk);  /* How far in future is RTO? */
    2767           0 :         if (rto_delta_us > 0)
    2768           0 :                 timeout = min_t(u32, timeout, usecs_to_jiffies(rto_delta_us));
    2769             : 
    2770           0 :         tcp_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout, TCP_RTO_MAX);
    2771           0 :         return true;
    2772             : }
    2773             : 
    2774             : /* Thanks to skb fast clones, we can detect if a prior transmit of
    2775             :  * a packet is still in a qdisc or driver queue.
    2776             :  * In this case, there is very little point doing a retransmit !
    2777             :  */
    2778           0 : static bool skb_still_in_host_queue(const struct sock *sk,
    2779             :                                     const struct sk_buff *skb)
    2780             : {
    2781           0 :         if (unlikely(skb_fclone_busy(sk, skb))) {
    2782           0 :                 NET_INC_STATS(sock_net(sk),
    2783             :                               LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
    2784           0 :                 return true;
    2785             :         }
    2786             :         return false;
    2787             : }
    2788             : 
    2789             : /* When probe timeout (PTO) fires, try send a new segment if possible, else
    2790             :  * retransmit the last segment.
    2791             :  */
    2792           0 : void tcp_send_loss_probe(struct sock *sk)
    2793             : {
    2794           0 :         struct tcp_sock *tp = tcp_sk(sk);
    2795           0 :         struct sk_buff *skb;
    2796           0 :         int pcount;
    2797           0 :         int mss = tcp_current_mss(sk);
    2798             : 
    2799             :         /* At most one outstanding TLP */
    2800           0 :         if (tp->tlp_high_seq)
    2801           0 :                 goto rearm_timer;
    2802             : 
    2803           0 :         tp->tlp_retrans = 0;
    2804           0 :         skb = tcp_send_head(sk);
    2805           0 :         if (skb && tcp_snd_wnd_test(tp, skb, mss)) {
    2806           0 :                 pcount = tp->packets_out;
    2807           0 :                 tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
    2808           0 :                 if (tp->packets_out > pcount)
    2809           0 :                         goto probe_sent;
    2810           0 :                 goto rearm_timer;
    2811             :         }
    2812           0 :         skb = skb_rb_last(&sk->tcp_rtx_queue);
    2813           0 :         if (unlikely(!skb)) {
    2814           0 :                 WARN_ONCE(tp->packets_out,
    2815             :                           "invalid inflight: %u state %u cwnd %u mss %d\n",
    2816             :                           tp->packets_out, sk->sk_state, tp->snd_cwnd, mss);
    2817           0 :                 inet_csk(sk)->icsk_pending = 0;
    2818           0 :                 return;
    2819             :         }
    2820             : 
    2821           0 :         if (skb_still_in_host_queue(sk, skb))
    2822           0 :                 goto rearm_timer;
    2823             : 
    2824           0 :         pcount = tcp_skb_pcount(skb);
    2825           0 :         if (WARN_ON(!pcount))
    2826           0 :                 goto rearm_timer;
    2827             : 
    2828           0 :         if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
    2829           0 :                 if (unlikely(tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
    2830             :                                           (pcount - 1) * mss, mss,
    2831             :                                           GFP_ATOMIC)))
    2832           0 :                         goto rearm_timer;
    2833           0 :                 skb = skb_rb_next(skb);
    2834             :         }
    2835             : 
    2836           0 :         if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
    2837           0 :                 goto rearm_timer;
    2838             : 
    2839           0 :         if (__tcp_retransmit_skb(sk, skb, 1))
    2840           0 :                 goto rearm_timer;
    2841             : 
    2842           0 :         tp->tlp_retrans = 1;
    2843             : 
    2844           0 : probe_sent:
    2845             :         /* Record snd_nxt for loss detection. */
    2846           0 :         tp->tlp_high_seq = tp->snd_nxt;
    2847             : 
    2848           0 :         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES);
    2849             :         /* Reset s.t. tcp_rearm_rto will restart timer from now */
    2850           0 :         inet_csk(sk)->icsk_pending = 0;
    2851           0 : rearm_timer:
    2852           0 :         tcp_rearm_rto(sk);
    2853             : }
    2854             : 
    2855             : /* Push out any pending frames which were held back due to
    2856             :  * TCP_CORK or attempt at coalescing tiny packets.
    2857             :  * The socket must be locked by the caller.
    2858             :  */
    2859         364 : void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
    2860             :                                int nonagle)
    2861             : {
    2862             :         /* If we are closed, the bytes will have to remain here.
    2863             :          * In time closedown will finish, we empty the write queue and
    2864             :          * all will be happy.
    2865             :          */
    2866         364 :         if (unlikely(sk->sk_state == TCP_CLOSE))
    2867             :                 return;
    2868             : 
    2869         364 :         if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
    2870             :                            sk_gfp_mask(sk, GFP_ATOMIC)))
    2871           0 :                 tcp_check_probe_timer(sk);
    2872             : }
    2873             : 
    2874             : /* Send _single_ skb sitting at the send head. This function requires
    2875             :  * true push pending frames to setup probe timer etc.
    2876             :  */
    2877           0 : void tcp_push_one(struct sock *sk, unsigned int mss_now)
    2878             : {
    2879           0 :         struct sk_buff *skb = tcp_send_head(sk);
    2880             : 
    2881           0 :         BUG_ON(!skb || skb->len < mss_now);
    2882             : 
    2883           0 :         tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
    2884           0 : }
    2885             : 
    2886             : /* This function returns the amount that we can raise the
    2887             :  * usable window based on the following constraints
    2888             :  *
    2889             :  * 1. The window can never be shrunk once it is offered (RFC 793)
    2890             :  * 2. We limit memory per socket
    2891             :  *
    2892             :  * RFC 1122:
    2893             :  * "the suggested [SWS] avoidance algorithm for the receiver is to keep
    2894             :  *  RECV.NEXT + RCV.WIN fixed until:
    2895             :  *  RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
    2896             :  *
    2897             :  * i.e. don't raise the right edge of the window until you can raise
    2898             :  * it at least MSS bytes.
    2899             :  *
    2900             :  * Unfortunately, the recommended algorithm breaks header prediction,
    2901             :  * since header prediction assumes th->window stays fixed.
    2902             :  *
    2903             :  * Strictly speaking, keeping th->window fixed violates the receiver
    2904             :  * side SWS prevention criteria. The problem is that under this rule
    2905             :  * a stream of single byte packets will cause the right side of the
    2906             :  * window to always advance by a single byte.
    2907             :  *
    2908             :  * Of course, if the sender implements sender side SWS prevention
    2909             :  * then this will not be a problem.
    2910             :  *
    2911             :  * BSD seems to make the following compromise:
    2912             :  *
    2913             :  *      If the free space is less than the 1/4 of the maximum
    2914             :  *      space available and the free space is less than 1/2 mss,
    2915             :  *      then set the window to 0.
    2916             :  *      [ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
    2917             :  *      Otherwise, just prevent the window from shrinking
    2918             :  *      and from being larger than the largest representable value.
    2919             :  *
    2920             :  * This prevents incremental opening of the window in the regime
    2921             :  * where TCP is limited by the speed of the reader side taking
    2922             :  * data out of the TCP receive queue. It does nothing about
    2923             :  * those cases where the window is constrained on the sender side
    2924             :  * because the pipeline is full.
    2925             :  *
    2926             :  * BSD also seems to "accidentally" limit itself to windows that are a
    2927             :  * multiple of MSS, at least until the free space gets quite small.
    2928             :  * This would appear to be a side effect of the mbuf implementation.
    2929             :  * Combining these two algorithms results in the observed behavior
    2930             :  * of having a fixed window size at almost all times.
    2931             :  *
    2932             :  * Below we obtain similar behavior by forcing the offered window to
    2933             :  * a multiple of the mss when it is feasible to do so.
    2934             :  *
    2935             :  * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
    2936             :  * Regular options like TIMESTAMP are taken into account.
    2937             :  */
    2938         464 : u32 __tcp_select_window(struct sock *sk)
    2939             : {
    2940         464 :         struct inet_connection_sock *icsk = inet_csk(sk);
    2941         464 :         struct tcp_sock *tp = tcp_sk(sk);
    2942             :         /* MSS for the peer's data.  Previous versions used mss_clamp
    2943             :          * here.  I don't know if the value based on our guesses
    2944             :          * of peer's MSS is better for the performance.  It's more correct
    2945             :          * but may be worse for the performance because of rcv_mss
    2946             :          * fluctuations.  --SAW  1998/11/1
    2947             :          */
    2948         464 :         int mss = icsk->icsk_ack.rcv_mss;
    2949         464 :         int free_space = tcp_space(sk);
    2950         464 :         int allowed_space = tcp_full_space(sk);
    2951         464 :         int full_space, window;
    2952             : 
    2953         464 :         if (sk_is_mptcp(sk))
    2954         464 :                 mptcp_space(sk, &free_space, &allowed_space);
    2955             : 
    2956         464 :         full_space = min_t(int, tp->window_clamp, allowed_space);
    2957             : 
    2958         464 :         if (unlikely(mss > full_space)) {
    2959           0 :                 mss = full_space;
    2960           0 :                 if (mss <= 0)
    2961             :                         return 0;
    2962             :         }
    2963         464 :         if (free_space < (full_space >> 1)) {
    2964           1 :                 icsk->icsk_ack.quick = 0;
    2965             : 
    2966           1 :                 if (tcp_under_memory_pressure(sk))
    2967           0 :                         tp->rcv_ssthresh = min(tp->rcv_ssthresh,
    2968             :                                                4U * tp->advmss);
    2969             : 
    2970             :                 /* free_space might become our new window, make sure we don't
    2971             :                  * increase it due to wscale.
    2972             :                  */
    2973           1 :                 free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
    2974             : 
    2975             :                 /* if free space is less than mss estimate, or is below 1/16th
    2976             :                  * of the maximum allowed, try to move to zero-window, else
    2977             :                  * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and
    2978             :                  * new incoming data is dropped due to memory limits.
    2979             :                  * With large window, mss test triggers way too late in order
    2980             :                  * to announce zero window in time before rmem limit kicks in.
    2981             :                  */
    2982           1 :                 if (free_space < (allowed_space >> 4) || free_space < mss)
    2983             :                         return 0;
    2984             :         }
    2985             : 
    2986         464 :         if (free_space > tp->rcv_ssthresh)
    2987         454 :                 free_space = tp->rcv_ssthresh;
    2988             : 
    2989             :         /* Don't do rounding if we are using window scaling, since the
    2990             :          * scaled window will not line up with the MSS boundary anyway.
    2991             :          */
    2992         464 :         if (tp->rx_opt.rcv_wscale) {
    2993           0 :                 window = free_space;
    2994             : 
    2995             :                 /* Advertise enough space so that it won't get scaled away.
    2996             :                  * Import case: prevent zero window announcement if
    2997             :                  * 1<<rcv_wscale > mss.
    2998             :                  */
    2999           0 :                 window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale));
    3000             :         } else {
    3001         464 :                 window = tp->rcv_wnd;
    3002             :                 /* Get the largest window that is a nice multiple of mss.
    3003             :                  * Window clamp already applied above.
    3004             :                  * If our current window offering is within 1 mss of the
    3005             :                  * free space we just keep it. This prevents the divide
    3006             :                  * and multiply from happening most of the time.
    3007             :                  * We also don't do any window rounding when the free space
    3008             :                  * is too small.
    3009             :                  */
    3010         464 :                 if (window <= free_space - mss || window > free_space)
    3011          42 :                         window = rounddown(free_space, mss);
    3012         422 :                 else if (mss == full_space &&
    3013           0 :                          free_space > window + (full_space >> 1))
    3014           0 :                         window = free_space;
    3015             :         }
    3016             : 
    3017         464 :         return window;
    3018             : }
    3019             : 
    3020           0 : void tcp_skb_collapse_tstamp(struct sk_buff *skb,
    3021             :                              const struct sk_buff *next_skb)
    3022             : {
    3023           0 :         if (unlikely(tcp_has_tx_tstamp(next_skb))) {
    3024           0 :                 const struct skb_shared_info *next_shinfo =
    3025           0 :                         skb_shinfo(next_skb);
    3026           0 :                 struct skb_shared_info *shinfo = skb_shinfo(skb);
    3027             : 
    3028           0 :                 shinfo->tx_flags |= next_shinfo->tx_flags & SKBTX_ANY_TSTAMP;
    3029           0 :                 shinfo->tskey = next_shinfo->tskey;
    3030           0 :                 TCP_SKB_CB(skb)->txstamp_ack |=
    3031           0 :                         TCP_SKB_CB(next_skb)->txstamp_ack;
    3032             :         }
    3033           0 : }
    3034             : 
    3035             : /* Collapses two adjacent SKB's during retransmission. */
    3036           0 : static bool tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
    3037             : {
    3038           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3039           0 :         struct sk_buff *next_skb = skb_rb_next(skb);
    3040           0 :         int next_skb_size;
    3041             : 
    3042           0 :         next_skb_size = next_skb->len;
    3043             : 
    3044           0 :         BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
    3045             : 
    3046           0 :         if (next_skb_size) {
    3047           0 :                 if (next_skb_size <= skb_availroom(skb))
    3048           0 :                         skb_copy_bits(next_skb, 0, skb_put(skb, next_skb_size),
    3049             :                                       next_skb_size);
    3050           0 :                 else if (!tcp_skb_shift(skb, next_skb, 1, next_skb_size))
    3051             :                         return false;
    3052             :         }
    3053           0 :         tcp_highest_sack_replace(sk, next_skb, skb);
    3054             : 
    3055             :         /* Update sequence range on original skb. */
    3056           0 :         TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
    3057             : 
    3058             :         /* Merge over control information. This moves PSH/FIN etc. over */
    3059           0 :         TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
    3060             : 
    3061             :         /* All done, get rid of second SKB and account for it so
    3062             :          * packet counting does not break.
    3063             :          */
    3064           0 :         TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
    3065           0 :         TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next_skb)->eor;
    3066             : 
    3067             :         /* changed transmit queue under us so clear hints */
    3068           0 :         tcp_clear_retrans_hints_partial(tp);
    3069           0 :         if (next_skb == tp->retransmit_skb_hint)
    3070           0 :                 tp->retransmit_skb_hint = skb;
    3071             : 
    3072           0 :         tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
    3073             : 
    3074           0 :         tcp_skb_collapse_tstamp(skb, next_skb);
    3075             : 
    3076           0 :         tcp_rtx_queue_unlink_and_free(next_skb, sk);
    3077           0 :         return true;
    3078             : }
    3079             : 
    3080             : /* Check if coalescing SKBs is legal. */
    3081           0 : static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
    3082             : {
    3083           0 :         if (tcp_skb_pcount(skb) > 1)
    3084             :                 return false;
    3085           0 :         if (skb_cloned(skb))
    3086             :                 return false;
    3087             :         /* Some heuristics for collapsing over SACK'd could be invented */
    3088           0 :         if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
    3089             :                 return false;
    3090             : 
    3091             :         return true;
    3092             : }
    3093             : 
    3094             : /* Collapse packets in the retransmit queue to make to create
    3095             :  * less packets on the wire. This is only done on retransmission.
    3096             :  */
    3097           0 : static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
    3098             :                                      int space)
    3099             : {
    3100           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3101           0 :         struct sk_buff *skb = to, *tmp;
    3102           0 :         bool first = true;
    3103             : 
    3104           0 :         if (!sock_net(sk)->ipv4.sysctl_tcp_retrans_collapse)
    3105             :                 return;
    3106           0 :         if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
    3107             :                 return;
    3108             : 
    3109           0 :         skb_rbtree_walk_from_safe(skb, tmp) {
    3110           0 :                 if (!tcp_can_collapse(sk, skb))
    3111             :                         break;
    3112             : 
    3113           0 :                 if (!tcp_skb_can_collapse(to, skb))
    3114             :                         break;
    3115             : 
    3116           0 :                 space -= skb->len;
    3117             : 
    3118           0 :                 if (first) {
    3119           0 :                         first = false;
    3120           0 :                         continue;
    3121             :                 }
    3122             : 
    3123           0 :                 if (space < 0)
    3124             :                         break;
    3125             : 
    3126           0 :                 if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
    3127             :                         break;
    3128             : 
    3129           0 :                 if (!tcp_collapse_retrans(sk, to))
    3130             :                         break;
    3131             :         }
    3132             : }
    3133             : 
    3134             : /* This retransmits one SKB.  Policy decisions and retransmit queue
    3135             :  * state updates are done by the caller.  Returns non-zero if an
    3136             :  * error occurred which prevented the send.
    3137             :  */
    3138           0 : int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
    3139             : {
    3140           0 :         struct inet_connection_sock *icsk = inet_csk(sk);
    3141           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3142           0 :         unsigned int cur_mss;
    3143           0 :         int diff, len, err;
    3144             : 
    3145             : 
    3146             :         /* Inconclusive MTU probe */
    3147           0 :         if (icsk->icsk_mtup.probe_size)
    3148           0 :                 icsk->icsk_mtup.probe_size = 0;
    3149             : 
    3150             :         /* Do not sent more than we queued. 1/4 is reserved for possible
    3151             :          * copying overhead: fragmentation, tunneling, mangling etc.
    3152             :          */
    3153           0 :         if (refcount_read(&sk->sk_wmem_alloc) >
    3154           0 :             min_t(u32, sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2),
    3155             :                   sk->sk_sndbuf))
    3156             :                 return -EAGAIN;
    3157             : 
    3158           0 :         if (skb_still_in_host_queue(sk, skb))
    3159             :                 return -EBUSY;
    3160             : 
    3161           0 :         if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
    3162           0 :                 if (unlikely(before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))) {
    3163           0 :                         WARN_ON_ONCE(1);
    3164           0 :                         return -EINVAL;
    3165             :                 }
    3166           0 :                 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
    3167             :                         return -ENOMEM;
    3168             :         }
    3169             : 
    3170           0 :         if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
    3171             :                 return -EHOSTUNREACH; /* Routing failure or similar. */
    3172             : 
    3173           0 :         cur_mss = tcp_current_mss(sk);
    3174             : 
    3175             :         /* If receiver has shrunk his window, and skb is out of
    3176             :          * new window, do not retransmit it. The exception is the
    3177             :          * case, when window is shrunk to zero. In this case
    3178             :          * our retransmit serves as a zero window probe.
    3179             :          */
    3180           0 :         if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) &&
    3181             :             TCP_SKB_CB(skb)->seq != tp->snd_una)
    3182             :                 return -EAGAIN;
    3183             : 
    3184           0 :         len = cur_mss * segs;
    3185           0 :         if (skb->len > len) {
    3186           0 :                 if (tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb, len,
    3187             :                                  cur_mss, GFP_ATOMIC))
    3188             :                         return -ENOMEM; /* We'll try again later. */
    3189             :         } else {
    3190           0 :                 if (skb_unclone(skb, GFP_ATOMIC))
    3191             :                         return -ENOMEM;
    3192             : 
    3193           0 :                 diff = tcp_skb_pcount(skb);
    3194           0 :                 tcp_set_skb_tso_segs(skb, cur_mss);
    3195           0 :                 diff -= tcp_skb_pcount(skb);
    3196           0 :                 if (diff)
    3197           0 :                         tcp_adjust_pcount(sk, skb, diff);
    3198           0 :                 if (skb->len < cur_mss)
    3199           0 :                         tcp_retrans_try_collapse(sk, skb, cur_mss);
    3200             :         }
    3201             : 
    3202             :         /* RFC3168, section 6.1.1.1. ECN fallback */
    3203           0 :         if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN_ECN) == TCPHDR_SYN_ECN)
    3204           0 :                 tcp_ecn_clear_syn(sk, skb);
    3205             : 
    3206             :         /* Update global and local TCP statistics. */
    3207           0 :         segs = tcp_skb_pcount(skb);
    3208           0 :         TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs);
    3209           0 :         if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
    3210           0 :                 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
    3211           0 :         tp->total_retrans += segs;
    3212           0 :         tp->bytes_retrans += skb->len;
    3213             : 
    3214             :         /* make sure skb->data is aligned on arches that require it
    3215             :          * and check if ack-trimming & collapsing extended the headroom
    3216             :          * beyond what csum_start can cover.
    3217             :          */
    3218           0 :         if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
    3219             :                      skb_headroom(skb) >= 0xFFFF)) {
    3220           0 :                 struct sk_buff *nskb;
    3221             : 
    3222           0 :                 tcp_skb_tsorted_save(skb) {
    3223           0 :                         nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC);
    3224           0 :                         if (nskb) {
    3225           0 :                                 nskb->dev = NULL;
    3226           0 :                                 err = tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC);
    3227             :                         } else {
    3228             :                                 err = -ENOBUFS;
    3229             :                         }
    3230           0 :                 } tcp_skb_tsorted_restore(skb);
    3231             : 
    3232           0 :                 if (!err) {
    3233           0 :                         tcp_update_skb_after_send(sk, skb, tp->tcp_wstamp_ns);
    3234           0 :                         tcp_rate_skb_sent(sk, skb);
    3235             :                 }
    3236             :         } else {
    3237           0 :                 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
    3238             :         }
    3239             : 
    3240             :         /* To avoid taking spuriously low RTT samples based on a timestamp
    3241             :          * for a transmit that never happened, always mark EVER_RETRANS
    3242             :          */
    3243           0 :         TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS;
    3244             : 
    3245           0 :         if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG))
    3246           0 :                 tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB,
    3247             :                                   TCP_SKB_CB(skb)->seq, segs, err);
    3248             : 
    3249           0 :         if (likely(!err)) {
    3250           0 :                 trace_tcp_retransmit_skb(sk, skb);
    3251           0 :         } else if (err != -EBUSY) {
    3252           0 :                 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs);
    3253             :         }
    3254             :         return err;
    3255             : }
    3256             : 
    3257           0 : int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
    3258             : {
    3259           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3260           0 :         int err = __tcp_retransmit_skb(sk, skb, segs);
    3261             : 
    3262           0 :         if (err == 0) {
    3263             : #if FASTRETRANS_DEBUG > 0
    3264           0 :                 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
    3265           0 :                         net_dbg_ratelimited("retrans_out leaked\n");
    3266             :                 }
    3267             : #endif
    3268           0 :                 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
    3269           0 :                 tp->retrans_out += tcp_skb_pcount(skb);
    3270             :         }
    3271             : 
    3272             :         /* Save stamp of the first (attempted) retransmit. */
    3273           0 :         if (!tp->retrans_stamp)
    3274           0 :                 tp->retrans_stamp = tcp_skb_timestamp(skb);
    3275             : 
    3276           0 :         if (tp->undo_retrans < 0)
    3277           0 :                 tp->undo_retrans = 0;
    3278           0 :         tp->undo_retrans += tcp_skb_pcount(skb);
    3279           0 :         return err;
    3280             : }
    3281             : 
    3282             : /* This gets called after a retransmit timeout, and the initially
    3283             :  * retransmitted data is acknowledged.  It tries to continue
    3284             :  * resending the rest of the retransmit queue, until either
    3285             :  * we've sent it all or the congestion window limit is reached.
    3286             :  */
    3287           0 : void tcp_xmit_retransmit_queue(struct sock *sk)
    3288             : {
    3289           0 :         const struct inet_connection_sock *icsk = inet_csk(sk);
    3290           0 :         struct sk_buff *skb, *rtx_head, *hole = NULL;
    3291           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3292           0 :         bool rearm_timer = false;
    3293           0 :         u32 max_segs;
    3294           0 :         int mib_idx;
    3295             : 
    3296           0 :         if (!tp->packets_out)
    3297             :                 return;
    3298             : 
    3299           0 :         rtx_head = tcp_rtx_queue_head(sk);
    3300           0 :         skb = tp->retransmit_skb_hint ?: rtx_head;
    3301           0 :         max_segs = tcp_tso_segs(sk, tcp_current_mss(sk));
    3302           0 :         skb_rbtree_walk_from(skb) {
    3303           0 :                 __u8 sacked;
    3304           0 :                 int segs;
    3305             : 
    3306           0 :                 if (tcp_pacing_check(sk))
    3307             :                         break;
    3308             : 
    3309             :                 /* we could do better than to assign each time */
    3310           0 :                 if (!hole)
    3311           0 :                         tp->retransmit_skb_hint = skb;
    3312             : 
    3313           0 :                 segs = tp->snd_cwnd - tcp_packets_in_flight(tp);
    3314           0 :                 if (segs <= 0)
    3315             :                         break;
    3316           0 :                 sacked = TCP_SKB_CB(skb)->sacked;
    3317             :                 /* In case tcp_shift_skb_data() have aggregated large skbs,
    3318             :                  * we need to make sure not sending too bigs TSO packets
    3319             :                  */
    3320           0 :                 segs = min_t(int, segs, max_segs);
    3321             : 
    3322           0 :                 if (tp->retrans_out >= tp->lost_out) {
    3323             :                         break;
    3324           0 :                 } else if (!(sacked & TCPCB_LOST)) {
    3325           0 :                         if (!hole && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
    3326           0 :                                 hole = skb;
    3327           0 :                         continue;
    3328             : 
    3329             :                 } else {
    3330           0 :                         if (icsk->icsk_ca_state != TCP_CA_Loss)
    3331             :                                 mib_idx = LINUX_MIB_TCPFASTRETRANS;
    3332             :                         else
    3333           0 :                                 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
    3334             :                 }
    3335             : 
    3336           0 :                 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
    3337           0 :                         continue;
    3338             : 
    3339           0 :                 if (tcp_small_queue_check(sk, skb, 1))
    3340             :                         break;
    3341             : 
    3342           0 :                 if (tcp_retransmit_skb(sk, skb, segs))
    3343             :                         break;
    3344             : 
    3345           0 :                 NET_ADD_STATS(sock_net(sk), mib_idx, tcp_skb_pcount(skb));
    3346             : 
    3347           0 :                 if (tcp_in_cwnd_reduction(sk))
    3348           0 :                         tp->prr_out += tcp_skb_pcount(skb);
    3349             : 
    3350           0 :                 if (skb == rtx_head &&
    3351           0 :                     icsk->icsk_pending != ICSK_TIME_REO_TIMEOUT)
    3352           0 :                         rearm_timer = true;
    3353             : 
    3354             :         }
    3355           0 :         if (rearm_timer)
    3356           0 :                 tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
    3357           0 :                                      inet_csk(sk)->icsk_rto,
    3358             :                                      TCP_RTO_MAX);
    3359             : }
    3360             : 
    3361             : /* We allow to exceed memory limits for FIN packets to expedite
    3362             :  * connection tear down and (memory) recovery.
    3363             :  * Otherwise tcp_send_fin() could be tempted to either delay FIN
    3364             :  * or even be forced to close flow without any FIN.
    3365             :  * In general, we want to allow one skb per socket to avoid hangs
    3366             :  * with edge trigger epoll()
    3367             :  */
    3368         228 : void sk_forced_mem_schedule(struct sock *sk, int size)
    3369             : {
    3370         228 :         int amt;
    3371             : 
    3372         228 :         if (size <= sk->sk_forward_alloc)
    3373             :                 return;
    3374          11 :         amt = sk_mem_pages(size);
    3375          11 :         sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
    3376          11 :         sk_memory_allocated_add(sk, amt);
    3377             : 
    3378          11 :         if (mem_cgroup_sockets_enabled && sk->sk_memcg)
    3379             :                 mem_cgroup_charge_skmem(sk->sk_memcg, amt);
    3380             : }
    3381             : 
    3382             : /* Send a FIN. The caller locks the socket for us.
    3383             :  * We should try to send a FIN packet really hard, but eventually give up.
    3384             :  */
    3385           3 : void tcp_send_fin(struct sock *sk)
    3386             : {
    3387           3 :         struct sk_buff *skb, *tskb, *tail = tcp_write_queue_tail(sk);
    3388           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3389             : 
    3390             :         /* Optimization, tack on the FIN if we have one skb in write queue and
    3391             :          * this skb was not yet sent, or we are under memory pressure.
    3392             :          * Note: in the latter case, FIN packet will be sent after a timeout,
    3393             :          * as TCP stack thinks it has already been transmitted.
    3394             :          */
    3395           3 :         tskb = tail;
    3396           3 :         if (!tskb && tcp_under_memory_pressure(sk))
    3397           0 :                 tskb = skb_rb_last(&sk->tcp_rtx_queue);
    3398             : 
    3399           3 :         if (tskb) {
    3400           0 :                 TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN;
    3401           0 :                 TCP_SKB_CB(tskb)->end_seq++;
    3402           0 :                 tp->write_seq++;
    3403           0 :                 if (!tail) {
    3404             :                         /* This means tskb was already sent.
    3405             :                          * Pretend we included the FIN on previous transmit.
    3406             :                          * We need to set tp->snd_nxt to the value it would have
    3407             :                          * if FIN had been sent. This is because retransmit path
    3408             :                          * does not change tp->snd_nxt.
    3409             :                          */
    3410           0 :                         WRITE_ONCE(tp->snd_nxt, tp->snd_nxt + 1);
    3411           0 :                         return;
    3412             :                 }
    3413             :         } else {
    3414           3 :                 skb = alloc_skb_fclone(MAX_TCP_HEADER, sk->sk_allocation);
    3415           3 :                 if (unlikely(!skb))
    3416             :                         return;
    3417             : 
    3418           3 :                 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
    3419           3 :                 skb_reserve(skb, MAX_TCP_HEADER);
    3420           3 :                 sk_forced_mem_schedule(sk, skb->truesize);
    3421             :                 /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
    3422           3 :                 tcp_init_nondata_skb(skb, tp->write_seq,
    3423             :                                      TCPHDR_ACK | TCPHDR_FIN);
    3424           3 :                 tcp_queue_skb(sk, skb);
    3425             :         }
    3426           3 :         __tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
    3427             : }
    3428             : 
    3429             : /* We get here when a process closes a file descriptor (either due to
    3430             :  * an explicit close() or as a byproduct of exit()'ing) and there
    3431             :  * was unread data in the receive queue.  This behavior is recommended
    3432             :  * by RFC 2525, section 2.17.  -DaveM
    3433             :  */
    3434           0 : void tcp_send_active_reset(struct sock *sk, gfp_t priority)
    3435             : {
    3436           0 :         struct sk_buff *skb;
    3437             : 
    3438           0 :         TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
    3439             : 
    3440             :         /* NOTE: No TCP options attached and we never retransmit this. */
    3441           0 :         skb = alloc_skb(MAX_TCP_HEADER, priority);
    3442           0 :         if (!skb) {
    3443           0 :                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
    3444           0 :                 return;
    3445             :         }
    3446             : 
    3447             :         /* Reserve space for headers and prepare control bits. */
    3448           0 :         skb_reserve(skb, MAX_TCP_HEADER);
    3449           0 :         tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
    3450             :                              TCPHDR_ACK | TCPHDR_RST);
    3451           0 :         tcp_mstamp_refresh(tcp_sk(sk));
    3452             :         /* Send it off. */
    3453           0 :         if (tcp_transmit_skb(sk, skb, 0, priority))
    3454           0 :                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
    3455             : 
    3456             :         /* skb of trace_tcp_send_reset() keeps the skb that caused RST,
    3457             :          * skb here is different to the troublesome skb, so use NULL
    3458             :          */
    3459           0 :         trace_tcp_send_reset(sk, NULL);
    3460             : }
    3461             : 
    3462             : /* Send a crossed SYN-ACK during socket establishment.
    3463             :  * WARNING: This routine must only be called when we have already sent
    3464             :  * a SYN packet that crossed the incoming SYN that caused this routine
    3465             :  * to get called. If this assumption fails then the initial rcv_wnd
    3466             :  * and rcv_wscale values will not be correct.
    3467             :  */
    3468           0 : int tcp_send_synack(struct sock *sk)
    3469             : {
    3470           0 :         struct sk_buff *skb;
    3471             : 
    3472           0 :         skb = tcp_rtx_queue_head(sk);
    3473           0 :         if (!skb || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
    3474           0 :                 pr_err("%s: wrong queue state\n", __func__);
    3475           0 :                 return -EFAULT;
    3476             :         }
    3477           0 :         if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
    3478           0 :                 if (skb_cloned(skb)) {
    3479           0 :                         struct sk_buff *nskb;
    3480             : 
    3481           0 :                         tcp_skb_tsorted_save(skb) {
    3482           0 :                                 nskb = skb_copy(skb, GFP_ATOMIC);
    3483           0 :                         } tcp_skb_tsorted_restore(skb);
    3484           0 :                         if (!nskb)
    3485             :                                 return -ENOMEM;
    3486           0 :                         INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor);
    3487           0 :                         tcp_highest_sack_replace(sk, skb, nskb);
    3488           0 :                         tcp_rtx_queue_unlink_and_free(skb, sk);
    3489           0 :                         __skb_header_release(nskb);
    3490           0 :                         tcp_rbtree_insert(&sk->tcp_rtx_queue, nskb);
    3491           0 :                         sk_wmem_queued_add(sk, nskb->truesize);
    3492           0 :                         sk_mem_charge(sk, nskb->truesize);
    3493             :                         skb = nskb;
    3494             :                 }
    3495             : 
    3496           0 :                 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
    3497           0 :                 tcp_ecn_send_synack(sk, skb);
    3498             :         }
    3499           0 :         return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
    3500             : }
    3501             : 
    3502             : /**
    3503             :  * tcp_make_synack - Allocate one skb and build a SYNACK packet.
    3504             :  * @sk: listener socket
    3505             :  * @dst: dst entry attached to the SYNACK. It is consumed and caller
    3506             :  *       should not use it again.
    3507             :  * @req: request_sock pointer
    3508             :  * @foc: cookie for tcp fast open
    3509             :  * @synack_type: Type of synack to prepare
    3510             :  * @syn_skb: SYN packet just received.  It could be NULL for rtx case.
    3511             :  */
    3512           4 : struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
    3513             :                                 struct request_sock *req,
    3514             :                                 struct tcp_fastopen_cookie *foc,
    3515             :                                 enum tcp_synack_type synack_type,
    3516             :                                 struct sk_buff *syn_skb)
    3517             : {
    3518           4 :         struct inet_request_sock *ireq = inet_rsk(req);
    3519           4 :         const struct tcp_sock *tp = tcp_sk(sk);
    3520           4 :         struct tcp_md5sig_key *md5 = NULL;
    3521           4 :         struct tcp_out_options opts;
    3522           4 :         struct sk_buff *skb;
    3523           4 :         int tcp_header_size;
    3524           4 :         struct tcphdr *th;
    3525           4 :         int mss;
    3526           4 :         u64 now;
    3527             : 
    3528           4 :         skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
    3529           4 :         if (unlikely(!skb)) {
    3530           0 :                 dst_release(dst);
    3531           0 :                 return NULL;
    3532             :         }
    3533             :         /* Reserve space for headers. */
    3534           4 :         skb_reserve(skb, MAX_TCP_HEADER);
    3535             : 
    3536           4 :         switch (synack_type) {
    3537             :         case TCP_SYNACK_NORMAL:
    3538           4 :                 skb_set_owner_w(skb, req_to_sk(req));
    3539           4 :                 break;
    3540             :         case TCP_SYNACK_COOKIE:
    3541             :                 /* Under synflood, we do not attach skb to a socket,
    3542             :                  * to avoid false sharing.
    3543             :                  */
    3544             :                 break;
    3545           0 :         case TCP_SYNACK_FASTOPEN:
    3546             :                 /* sk is a const pointer, because we want to express multiple
    3547             :                  * cpu might call us concurrently.
    3548             :                  * sk->sk_wmem_alloc in an atomic, we can promote to rw.
    3549             :                  */
    3550           0 :                 skb_set_owner_w(skb, (struct sock *)sk);
    3551           0 :                 break;
    3552             :         }
    3553           4 :         skb_dst_set(skb, dst);
    3554             : 
    3555           4 :         mss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
    3556             : 
    3557           4 :         memset(&opts, 0, sizeof(opts));
    3558           4 :         now = tcp_clock_ns();
    3559             : #ifdef CONFIG_SYN_COOKIES
    3560             :         if (unlikely(synack_type == TCP_SYNACK_COOKIE && ireq->tstamp_ok))
    3561             :                 skb->skb_mstamp_ns = cookie_init_timestamp(req, now);
    3562             :         else
    3563             : #endif
    3564             :         {
    3565           4 :                 skb->skb_mstamp_ns = now;
    3566           4 :                 if (!tcp_rsk(req)->snt_synack) /* Timestamp first SYNACK */
    3567           4 :                         tcp_rsk(req)->snt_synack = tcp_skb_timestamp_us(skb);
    3568             :         }
    3569             : 
    3570             : #ifdef CONFIG_TCP_MD5SIG
    3571             :         rcu_read_lock();
    3572             :         md5 = tcp_rsk(req)->af_specific->req_md5_lookup(sk, req_to_sk(req));
    3573             : #endif
    3574           4 :         skb_set_hash(skb, tcp_rsk(req)->txhash, PKT_HASH_TYPE_L4);
    3575             :         /* bpf program will be interested in the tcp_flags */
    3576           4 :         TCP_SKB_CB(skb)->tcp_flags = TCPHDR_SYN | TCPHDR_ACK;
    3577           4 :         tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts, md5,
    3578             :                                              foc, synack_type,
    3579           4 :                                              syn_skb) + sizeof(*th);
    3580             : 
    3581           4 :         skb_push(skb, tcp_header_size);
    3582           4 :         skb_reset_transport_header(skb);
    3583             : 
    3584           4 :         th = (struct tcphdr *)skb->data;
    3585           4 :         memset(th, 0, sizeof(struct tcphdr));
    3586           4 :         th->syn = 1;
    3587           4 :         th->ack = 1;
    3588           4 :         tcp_ecn_make_synack(req, th);
    3589           4 :         th->source = htons(ireq->ir_num);
    3590           4 :         th->dest = ireq->ir_rmt_port;
    3591           4 :         skb->mark = ireq->ir_mark;
    3592           4 :         skb->ip_summed = CHECKSUM_PARTIAL;
    3593           4 :         th->seq = htonl(tcp_rsk(req)->snt_isn);
    3594             :         /* XXX data is queued and acked as is. No buffer/window check */
    3595           4 :         th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
    3596             : 
    3597             :         /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
    3598           4 :         th->window = htons(min(req->rsk_rcv_wnd, 65535U));
    3599           4 :         tcp_options_write((__be32 *)(th + 1), NULL, &opts);
    3600           4 :         th->doff = (tcp_header_size >> 2);
    3601           4 :         __TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
    3602             : 
    3603             : #ifdef CONFIG_TCP_MD5SIG
    3604             :         /* Okay, we have all we need - do the md5 hash if needed */
    3605             :         if (md5)
    3606             :                 tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
    3607             :                                                md5, req_to_sk(req), skb);
    3608             :         rcu_read_unlock();
    3609             : #endif
    3610             : 
    3611           4 :         bpf_skops_write_hdr_opt((struct sock *)sk, skb, req, syn_skb,
    3612             :                                 synack_type, &opts);
    3613             : 
    3614           4 :         skb->skb_mstamp_ns = now;
    3615           4 :         tcp_add_tx_delay(skb, tp);
    3616             : 
    3617           4 :         return skb;
    3618             : }
    3619             : EXPORT_SYMBOL(tcp_make_synack);
    3620             : 
    3621           0 : static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst)
    3622             : {
    3623           0 :         struct inet_connection_sock *icsk = inet_csk(sk);
    3624           0 :         const struct tcp_congestion_ops *ca;
    3625           0 :         u32 ca_key = dst_metric(dst, RTAX_CC_ALGO);
    3626             : 
    3627           0 :         if (ca_key == TCP_CA_UNSPEC)
    3628             :                 return;
    3629             : 
    3630           0 :         rcu_read_lock();
    3631           0 :         ca = tcp_ca_find_key(ca_key);
    3632           0 :         if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
    3633           0 :                 bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
    3634           0 :                 icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
    3635           0 :                 icsk->icsk_ca_ops = ca;
    3636             :         }
    3637           0 :         rcu_read_unlock();
    3638             : }
    3639             : 
    3640             : /* Do all connect socket setups that can be done AF independent. */
    3641           0 : static void tcp_connect_init(struct sock *sk)
    3642             : {
    3643           0 :         const struct dst_entry *dst = __sk_dst_get(sk);
    3644           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3645           0 :         __u8 rcv_wscale;
    3646           0 :         u32 rcv_wnd;
    3647             : 
    3648             :         /* We'll fix this up when we get a response from the other end.
    3649             :          * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
    3650             :          */
    3651           0 :         tp->tcp_header_len = sizeof(struct tcphdr);
    3652           0 :         if (sock_net(sk)->ipv4.sysctl_tcp_timestamps)
    3653           0 :                 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
    3654             : 
    3655             : #ifdef CONFIG_TCP_MD5SIG
    3656             :         if (tp->af_specific->md5_lookup(sk, sk))
    3657             :                 tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
    3658             : #endif
    3659             : 
    3660             :         /* If user gave his TCP_MAXSEG, record it to clamp */
    3661           0 :         if (tp->rx_opt.user_mss)
    3662           0 :                 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
    3663           0 :         tp->max_window = 0;
    3664           0 :         tcp_mtup_init(sk);
    3665           0 :         tcp_sync_mss(sk, dst_mtu(dst));
    3666             : 
    3667           0 :         tcp_ca_dst_init(sk, dst);
    3668             : 
    3669           0 :         if (!tp->window_clamp)
    3670           0 :                 tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
    3671           0 :         tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
    3672             : 
    3673           0 :         tcp_initialize_rcv_mss(sk);
    3674             : 
    3675             :         /* limit the window selection if the user enforce a smaller rx buffer */
    3676           0 :         if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
    3677           0 :             (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
    3678           0 :                 tp->window_clamp = tcp_full_space(sk);
    3679             : 
    3680           0 :         rcv_wnd = tcp_rwnd_init_bpf(sk);
    3681           0 :         if (rcv_wnd == 0)
    3682           0 :                 rcv_wnd = dst_metric(dst, RTAX_INITRWND);
    3683             : 
    3684           0 :         tcp_select_initial_window(sk, tcp_full_space(sk),
    3685           0 :                                   tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
    3686           0 :                                   &tp->rcv_wnd,
    3687           0 :                                   &tp->window_clamp,
    3688           0 :                                   sock_net(sk)->ipv4.sysctl_tcp_window_scaling,
    3689             :                                   &rcv_wscale,
    3690             :                                   rcv_wnd);
    3691             : 
    3692           0 :         tp->rx_opt.rcv_wscale = rcv_wscale;
    3693           0 :         tp->rcv_ssthresh = tp->rcv_wnd;
    3694             : 
    3695           0 :         sk->sk_err = 0;
    3696           0 :         sock_reset_flag(sk, SOCK_DONE);
    3697           0 :         tp->snd_wnd = 0;
    3698           0 :         tcp_init_wl(tp, 0);
    3699           0 :         tcp_write_queue_purge(sk);
    3700           0 :         tp->snd_una = tp->write_seq;
    3701           0 :         tp->snd_sml = tp->write_seq;
    3702           0 :         tp->snd_up = tp->write_seq;
    3703           0 :         WRITE_ONCE(tp->snd_nxt, tp->write_seq);
    3704             : 
    3705           0 :         if (likely(!tp->repair))
    3706           0 :                 tp->rcv_nxt = 0;
    3707             :         else
    3708           0 :                 tp->rcv_tstamp = tcp_jiffies32;
    3709           0 :         tp->rcv_wup = tp->rcv_nxt;
    3710           0 :         WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
    3711             : 
    3712           0 :         inet_csk(sk)->icsk_rto = tcp_timeout_init(sk);
    3713           0 :         inet_csk(sk)->icsk_retransmits = 0;
    3714           0 :         tcp_clear_retrans(tp);
    3715           0 : }
    3716             : 
    3717           0 : static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
    3718             : {
    3719           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3720           0 :         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
    3721             : 
    3722           0 :         tcb->end_seq += skb->len;
    3723           0 :         __skb_header_release(skb);
    3724           0 :         sk_wmem_queued_add(sk, skb->truesize);
    3725           0 :         sk_mem_charge(sk, skb->truesize);
    3726           0 :         WRITE_ONCE(tp->write_seq, tcb->end_seq);
    3727           0 :         tp->packets_out += tcp_skb_pcount(skb);
    3728           0 : }
    3729             : 
    3730             : /* Build and send a SYN with data and (cached) Fast Open cookie. However,
    3731             :  * queue a data-only packet after the regular SYN, such that regular SYNs
    3732             :  * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges
    3733             :  * only the SYN sequence, the data are retransmitted in the first ACK.
    3734             :  * If cookie is not cached or other error occurs, falls back to send a
    3735             :  * regular SYN with Fast Open cookie request option.
    3736             :  */
    3737           0 : static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
    3738             : {
    3739           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3740           0 :         struct tcp_fastopen_request *fo = tp->fastopen_req;
    3741           0 :         int space, err = 0;
    3742           0 :         struct sk_buff *syn_data;
    3743             : 
    3744           0 :         tp->rx_opt.mss_clamp = tp->advmss;  /* If MSS is not cached */
    3745           0 :         if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie))
    3746           0 :                 goto fallback;
    3747             : 
    3748             :         /* MSS for SYN-data is based on cached MSS and bounded by PMTU and
    3749             :          * user-MSS. Reserve maximum option space for middleboxes that add
    3750             :          * private TCP options. The cost is reduced data space in SYN :(
    3751             :          */
    3752           0 :         tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp);
    3753             : 
    3754           0 :         space = __tcp_mtu_to_mss(sk, inet_csk(sk)->icsk_pmtu_cookie) -
    3755             :                 MAX_TCP_OPTION_SPACE;
    3756             : 
    3757           0 :         space = min_t(size_t, space, fo->size);
    3758             : 
    3759             :         /* limit to order-0 allocations */
    3760           0 :         space = min_t(size_t, space, SKB_MAX_HEAD(MAX_TCP_HEADER));
    3761             : 
    3762           0 :         syn_data = sk_stream_alloc_skb(sk, space, sk->sk_allocation, false);
    3763           0 :         if (!syn_data)
    3764           0 :                 goto fallback;
    3765           0 :         syn_data->ip_summed = CHECKSUM_PARTIAL;
    3766           0 :         memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
    3767           0 :         if (space) {
    3768           0 :                 int copied = copy_from_iter(skb_put(syn_data, space), space,
    3769           0 :                                             &fo->data->msg_iter);
    3770           0 :                 if (unlikely(!copied)) {
    3771           0 :                         tcp_skb_tsorted_anchor_cleanup(syn_data);
    3772           0 :                         kfree_skb(syn_data);
    3773           0 :                         goto fallback;
    3774             :                 }
    3775           0 :                 if (copied != space) {
    3776           0 :                         skb_trim(syn_data, copied);
    3777           0 :                         space = copied;
    3778             :                 }
    3779           0 :                 skb_zcopy_set(syn_data, fo->uarg, NULL);
    3780             :         }
    3781             :         /* No more data pending in inet_wait_for_connect() */
    3782           0 :         if (space == fo->size)
    3783           0 :                 fo->data = NULL;
    3784           0 :         fo->copied = space;
    3785             : 
    3786           0 :         tcp_connect_queue_skb(sk, syn_data);
    3787           0 :         if (syn_data->len)
    3788           0 :                 tcp_chrono_start(sk, TCP_CHRONO_BUSY);
    3789             : 
    3790           0 :         err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation);
    3791             : 
    3792           0 :         syn->skb_mstamp_ns = syn_data->skb_mstamp_ns;
    3793             : 
    3794             :         /* Now full SYN+DATA was cloned and sent (or not),
    3795             :          * remove the SYN from the original skb (syn_data)
    3796             :          * we keep in write queue in case of a retransmit, as we
    3797             :          * also have the SYN packet (with no data) in the same queue.
    3798             :          */
    3799           0 :         TCP_SKB_CB(syn_data)->seq++;
    3800           0 :         TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH;
    3801           0 :         if (!err) {
    3802           0 :                 tp->syn_data = (fo->copied > 0);
    3803           0 :                 tcp_rbtree_insert(&sk->tcp_rtx_queue, syn_data);
    3804           0 :                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT);
    3805           0 :                 goto done;
    3806             :         }
    3807             : 
    3808             :         /* data was not sent, put it in write_queue */
    3809           0 :         __skb_queue_tail(&sk->sk_write_queue, syn_data);
    3810           0 :         tp->packets_out -= tcp_skb_pcount(syn_data);
    3811             : 
    3812           0 : fallback:
    3813             :         /* Send a regular SYN with Fast Open cookie request option */
    3814           0 :         if (fo->cookie.len > 0)
    3815           0 :                 fo->cookie.len = 0;
    3816           0 :         err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
    3817           0 :         if (err)
    3818           0 :                 tp->syn_fastopen = 0;
    3819           0 : done:
    3820           0 :         fo->cookie.len = -1;  /* Exclude Fast Open option for SYN retries */
    3821           0 :         return err;
    3822             : }
    3823             : 
    3824             : /* Build a SYN and send it off. */
    3825           0 : int tcp_connect(struct sock *sk)
    3826             : {
    3827           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3828           0 :         struct sk_buff *buff;
    3829           0 :         int err;
    3830             : 
    3831           0 :         tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL);
    3832             : 
    3833           0 :         if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
    3834             :                 return -EHOSTUNREACH; /* Routing failure or similar. */
    3835             : 
    3836           0 :         tcp_connect_init(sk);
    3837             : 
    3838           0 :         if (unlikely(tp->repair)) {
    3839           0 :                 tcp_finish_connect(sk, NULL);
    3840           0 :                 return 0;
    3841             :         }
    3842             : 
    3843           0 :         buff = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, true);
    3844           0 :         if (unlikely(!buff))
    3845             :                 return -ENOBUFS;
    3846             : 
    3847           0 :         tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
    3848           0 :         tcp_mstamp_refresh(tp);
    3849           0 :         tp->retrans_stamp = tcp_time_stamp(tp);
    3850           0 :         tcp_connect_queue_skb(sk, buff);
    3851           0 :         tcp_ecn_send_syn(sk, buff);
    3852           0 :         tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
    3853             : 
    3854             :         /* Send off SYN; include data in Fast Open. */
    3855           0 :         err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
    3856           0 :               tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
    3857           0 :         if (err == -ECONNREFUSED)
    3858             :                 return err;
    3859             : 
    3860             :         /* We change tp->snd_nxt after the tcp_transmit_skb() call
    3861             :          * in order to make this packet get counted in tcpOutSegs.
    3862             :          */
    3863           0 :         WRITE_ONCE(tp->snd_nxt, tp->write_seq);
    3864           0 :         tp->pushed_seq = tp->write_seq;
    3865           0 :         buff = tcp_send_head(sk);
    3866           0 :         if (unlikely(buff)) {
    3867           0 :                 WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(buff)->seq);
    3868           0 :                 tp->pushed_seq       = TCP_SKB_CB(buff)->seq;
    3869             :         }
    3870           0 :         TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
    3871             : 
    3872             :         /* Timer for repeating the SYN until an answer. */
    3873           0 :         inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
    3874           0 :                                   inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
    3875           0 :         return 0;
    3876             : }
    3877             : EXPORT_SYMBOL(tcp_connect);
    3878             : 
    3879             : /* Send out a delayed ack, the caller does the policy checking
    3880             :  * to see if we should even be here.  See tcp_input.c:tcp_ack_snd_check()
    3881             :  * for details.
    3882             :  */
    3883          14 : void tcp_send_delayed_ack(struct sock *sk)
    3884             : {
    3885          14 :         struct inet_connection_sock *icsk = inet_csk(sk);
    3886          14 :         int ato = icsk->icsk_ack.ato;
    3887          14 :         unsigned long timeout;
    3888             : 
    3889          14 :         if (ato > TCP_DELACK_MIN) {
    3890           1 :                 const struct tcp_sock *tp = tcp_sk(sk);
    3891           1 :                 int max_ato = HZ / 2;
    3892             : 
    3893           1 :                 if (inet_csk_in_pingpong_mode(sk) ||
    3894           0 :                     (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
    3895           1 :                         max_ato = TCP_DELACK_MAX;
    3896             : 
    3897             :                 /* Slow path, intersegment interval is "high". */
    3898             : 
    3899             :                 /* If some rtt estimate is known, use it to bound delayed ack.
    3900             :                  * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
    3901             :                  * directly.
    3902             :                  */
    3903           1 :                 if (tp->srtt_us) {
    3904           1 :                         int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),
    3905             :                                         TCP_DELACK_MIN);
    3906             : 
    3907           1 :                         if (rtt < max_ato)
    3908             :                                 max_ato = rtt;
    3909             :                 }
    3910             : 
    3911           1 :                 ato = min(ato, max_ato);
    3912             :         }
    3913             : 
    3914          14 :         ato = min_t(u32, ato, inet_csk(sk)->icsk_delack_max);
    3915             : 
    3916             :         /* Stay within the limit we were given */
    3917          14 :         timeout = jiffies + ato;
    3918             : 
    3919             :         /* Use new timeout only if there wasn't a older one earlier. */
    3920          14 :         if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
    3921             :                 /* If delack timer is about to expire, send ACK now. */
    3922           2 :                 if (time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
    3923           0 :                         tcp_send_ack(sk);
    3924           0 :                         return;
    3925             :                 }
    3926             : 
    3927           2 :                 if (!time_before(timeout, icsk->icsk_ack.timeout))
    3928           2 :                         timeout = icsk->icsk_ack.timeout;
    3929             :         }
    3930          14 :         icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
    3931          14 :         icsk->icsk_ack.timeout = timeout;
    3932          14 :         sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
    3933             : }
    3934             : 
    3935             : /* This routine sends an ack and also updates the window. */
    3936          62 : void __tcp_send_ack(struct sock *sk, u32 rcv_nxt)
    3937             : {
    3938          62 :         struct sk_buff *buff;
    3939             : 
    3940             :         /* If we have been reset, we may not send again. */
    3941          62 :         if (sk->sk_state == TCP_CLOSE)
    3942             :                 return;
    3943             : 
    3944             :         /* We are not putting this on the write queue, so
    3945             :          * tcp_transmit_skb() will set the ownership to this
    3946             :          * sock.
    3947             :          */
    3948          62 :         buff = alloc_skb(MAX_TCP_HEADER,
    3949             :                          sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
    3950          62 :         if (unlikely(!buff)) {
    3951           0 :                 struct inet_connection_sock *icsk = inet_csk(sk);
    3952           0 :                 unsigned long delay;
    3953             : 
    3954           0 :                 delay = TCP_DELACK_MAX << icsk->icsk_ack.retry;
    3955           0 :                 if (delay < TCP_RTO_MAX)
    3956           0 :                         icsk->icsk_ack.retry++;
    3957           0 :                 inet_csk_schedule_ack(sk);
    3958           0 :                 icsk->icsk_ack.ato = TCP_ATO_MIN;
    3959           0 :                 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, delay, TCP_RTO_MAX);
    3960           0 :                 return;
    3961             :         }
    3962             : 
    3963             :         /* Reserve space for headers and prepare control bits. */
    3964          62 :         skb_reserve(buff, MAX_TCP_HEADER);
    3965          62 :         tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);
    3966             : 
    3967             :         /* We do not want pure acks influencing TCP Small Queues or fq/pacing
    3968             :          * too much.
    3969             :          * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
    3970             :          */
    3971          62 :         skb_set_tcp_pure_ack(buff);
    3972             : 
    3973             :         /* Send it off, this clears delayed acks for us. */
    3974          62 :         __tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0, rcv_nxt);
    3975             : }
    3976             : EXPORT_SYMBOL_GPL(__tcp_send_ack);
    3977             : 
    3978          62 : void tcp_send_ack(struct sock *sk)
    3979             : {
    3980          62 :         __tcp_send_ack(sk, tcp_sk(sk)->rcv_nxt);
    3981          62 : }
    3982             : 
    3983             : /* This routine sends a packet with an out of date sequence
    3984             :  * number. It assumes the other end will try to ack it.
    3985             :  *
    3986             :  * Question: what should we make while urgent mode?
    3987             :  * 4.4BSD forces sending single byte of data. We cannot send
    3988             :  * out of window data, because we have SND.NXT==SND.MAX...
    3989             :  *
    3990             :  * Current solution: to send TWO zero-length segments in urgent mode:
    3991             :  * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
    3992             :  * out-of-date with SND.UNA-1 to probe window.
    3993             :  */
    3994           0 : static int tcp_xmit_probe_skb(struct sock *sk, int urgent, int mib)
    3995             : {
    3996           0 :         struct tcp_sock *tp = tcp_sk(sk);
    3997           0 :         struct sk_buff *skb;
    3998             : 
    3999             :         /* We don't queue it, tcp_transmit_skb() sets ownership. */
    4000           0 :         skb = alloc_skb(MAX_TCP_HEADER,
    4001             :                         sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
    4002           0 :         if (!skb)
    4003             :                 return -1;
    4004             : 
    4005             :         /* Reserve space for headers and set control bits. */
    4006           0 :         skb_reserve(skb, MAX_TCP_HEADER);
    4007             :         /* Use a previous sequence.  This should cause the other
    4008             :          * end to send an ack.  Don't queue or clone SKB, just
    4009             :          * send it.
    4010             :          */
    4011           0 :         tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);
    4012           0 :         NET_INC_STATS(sock_net(sk), mib);
    4013           0 :         return tcp_transmit_skb(sk, skb, 0, (__force gfp_t)0);
    4014             : }
    4015             : 
    4016             : /* Called from setsockopt( ... TCP_REPAIR ) */
    4017           0 : void tcp_send_window_probe(struct sock *sk)
    4018             : {
    4019           0 :         if (sk->sk_state == TCP_ESTABLISHED) {
    4020           0 :                 tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
    4021           0 :                 tcp_mstamp_refresh(tcp_sk(sk));
    4022           0 :                 tcp_xmit_probe_skb(sk, 0, LINUX_MIB_TCPWINPROBE);
    4023             :         }
    4024           0 : }
    4025             : 
    4026             : /* Initiate keepalive or window probe from timer. */
    4027           0 : int tcp_write_wakeup(struct sock *sk, int mib)
    4028             : {
    4029           0 :         struct tcp_sock *tp = tcp_sk(sk);
    4030           0 :         struct sk_buff *skb;
    4031             : 
    4032           0 :         if (sk->sk_state == TCP_CLOSE)
    4033             :                 return -1;
    4034             : 
    4035           0 :         skb = tcp_send_head(sk);
    4036           0 :         if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
    4037           0 :                 int err;
    4038           0 :                 unsigned int mss = tcp_current_mss(sk);
    4039           0 :                 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
    4040             : 
    4041           0 :                 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
    4042           0 :                         tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
    4043             : 
    4044             :                 /* We are probing the opening of a window
    4045             :                  * but the window size is != 0
    4046             :                  * must have been a result SWS avoidance ( sender )
    4047             :                  */
    4048           0 :                 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
    4049           0 :                     skb->len > mss) {
    4050           0 :                         seg_size = min(seg_size, mss);
    4051           0 :                         TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
    4052           0 :                         if (tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
    4053             :                                          skb, seg_size, mss, GFP_ATOMIC))
    4054             :                                 return -1;
    4055           0 :                 } else if (!tcp_skb_pcount(skb))
    4056           0 :                         tcp_set_skb_tso_segs(skb, mss);
    4057             : 
    4058           0 :                 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
    4059           0 :                 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
    4060           0 :                 if (!err)
    4061           0 :                         tcp_event_new_data_sent(sk, skb);
    4062           0 :                 return err;
    4063             :         } else {
    4064           0 :                 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
    4065           0 :                         tcp_xmit_probe_skb(sk, 1, mib);
    4066           0 :                 return tcp_xmit_probe_skb(sk, 0, mib);
    4067             :         }
    4068             : }
    4069             : 
    4070             : /* A window probe timeout has occurred.  If window is not closed send
    4071             :  * a partial packet else a zero probe.
    4072             :  */
    4073           0 : void tcp_send_probe0(struct sock *sk)
    4074             : {
    4075           0 :         struct inet_connection_sock *icsk = inet_csk(sk);
    4076           0 :         struct tcp_sock *tp = tcp_sk(sk);
    4077           0 :         struct net *net = sock_net(sk);
    4078           0 :         unsigned long timeout;
    4079           0 :         int err;
    4080             : 
    4081           0 :         err = tcp_write_wakeup(sk, LINUX_MIB_TCPWINPROBE);
    4082             : 
    4083           0 :         if (tp->packets_out || tcp_write_queue_empty(sk)) {
    4084             :                 /* Cancel probe timer, if it is not required. */
    4085           0 :                 icsk->icsk_probes_out = 0;
    4086           0 :                 icsk->icsk_backoff = 0;
    4087           0 :                 icsk->icsk_probes_tstamp = 0;
    4088           0 :                 return;
    4089             :         }
    4090             : 
    4091           0 :         icsk->icsk_probes_out++;
    4092           0 :         if (err <= 0) {
    4093           0 :                 if (icsk->icsk_backoff < net->ipv4.sysctl_tcp_retries2)
    4094           0 :                         icsk->icsk_backoff++;
    4095           0 :                 timeout = tcp_probe0_when(sk, TCP_RTO_MAX);
    4096             :         } else {
    4097             :                 /* If packet was not sent due to local congestion,
    4098             :                  * Let senders fight for local resources conservatively.
    4099             :                  */
    4100             :                 timeout = TCP_RESOURCE_PROBE_INTERVAL;
    4101             :         }
    4102             : 
    4103           0 :         timeout = tcp_clamp_probe0_to_user_timeout(sk, timeout);
    4104           0 :         tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, timeout, TCP_RTO_MAX);
    4105             : }
    4106             : 
    4107           0 : int tcp_rtx_synack(const struct sock *sk, struct request_sock *req)
    4108             : {
    4109           0 :         const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific;
    4110           0 :         struct flowi fl;
    4111           0 :         int res;
    4112             : 
    4113           0 :         tcp_rsk(req)->txhash = net_tx_rndhash();
    4114           0 :         res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL,
    4115             :                                   NULL);
    4116           0 :         if (!res) {
    4117           0 :                 __TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
    4118           0 :                 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
    4119           0 :                 if (unlikely(tcp_passive_fastopen(sk)))
    4120           0 :                         tcp_sk(sk)->total_retrans++;
    4121           0 :                 trace_tcp_retransmit_synack(sk, req);
    4122             :         }
    4123           0 :         return res;
    4124             : }
    4125             : EXPORT_SYMBOL(tcp_rtx_synack);

Generated by: LCOV version 1.14