Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : #ifndef _ASM_X86_CHECKSUM_64_H
3 : #define _ASM_X86_CHECKSUM_64_H
4 :
5 : /*
6 : * Checksums for x86-64
7 : * Copyright 2002 by Andi Kleen, SuSE Labs
8 : * with some code from asm-x86/checksum.h
9 : */
10 :
11 : #include <linux/compiler.h>
12 : #include <linux/uaccess.h>
13 : #include <asm/byteorder.h>
14 :
15 : /**
16 : * csum_fold - Fold and invert a 32bit checksum.
17 : * sum: 32bit unfolded sum
18 : *
19 : * Fold a 32bit running checksum to 16bit and invert it. This is usually
20 : * the last step before putting a checksum into a packet.
21 : * Make sure not to mix with 64bit checksums.
22 : */
23 1670 : static inline __sum16 csum_fold(__wsum sum)
24 : {
25 1670 : asm(" addl %1,%0\n"
26 : " adcl $0xffff,%0"
27 : : "=r" (sum)
28 1670 : : "r" ((__force u32)sum << 16),
29 1670 : "0" ((__force u32)sum & 0xffff0000));
30 935 : return (__force __sum16)(~(__force u32)sum >> 16);
31 : }
32 :
33 : /*
34 : * This is a version of ip_compute_csum() optimized for IP headers,
35 : * which always checksum on 4 octet boundaries.
36 : *
37 : * By Jorge Cwik <jorge@laser.satlink.net>, adapted for linux by
38 : * Arnt Gulbrandsen.
39 : */
40 :
41 : /**
42 : * ip_fast_csum - Compute the IPv4 header checksum efficiently.
43 : * iph: ipv4 header
44 : * ihl: length of header / 4
45 : */
46 1605 : static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
47 : {
48 1605 : unsigned int sum;
49 :
50 1605 : asm(" movl (%1), %0\n"
51 : " subl $4, %2\n"
52 : " jbe 2f\n"
53 : " addl 4(%1), %0\n"
54 : " adcl 8(%1), %0\n"
55 : " adcl 12(%1), %0\n"
56 : "1: adcl 16(%1), %0\n"
57 : " lea 4(%1), %1\n"
58 : " decl %2\n"
59 : " jne 1b\n"
60 : " adcl $0, %0\n"
61 : " movl %0, %2\n"
62 : " shrl $16, %0\n"
63 : " addw %w2, %w0\n"
64 : " adcl $0, %0\n"
65 : " notl %0\n"
66 : "2:"
67 : /* Since the input registers which are loaded with iph and ihl
68 : are modified, we must also specify them as outputs, or gcc
69 : will assume they contain their original values. */
70 : : "=r" (sum), "=r" (iph), "=r" (ihl)
71 : : "1" (iph), "2" (ihl)
72 : : "memory");
73 1605 : return (__force __sum16)sum;
74 : }
75 :
76 : /**
77 : * csum_tcpup_nofold - Compute an IPv4 pseudo header checksum.
78 : * @saddr: source address
79 : * @daddr: destination address
80 : * @len: length of packet
81 : * @proto: ip protocol of packet
82 : * @sum: initial sum to be added in (32bit unfolded)
83 : *
84 : * Returns the pseudo header checksum the input data. Result is
85 : * 32bit unfolded.
86 : */
87 : static inline __wsum
88 1166 : csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len,
89 : __u8 proto, __wsum sum)
90 : {
91 1166 : asm(" addl %1, %0\n"
92 : " adcl %2, %0\n"
93 : " adcl %3, %0\n"
94 : " adcl $0, %0\n"
95 : : "=r" (sum)
96 : : "g" (daddr), "g" (saddr),
97 1166 : "g" ((len + proto)<<8), "0" (sum));
98 736 : return sum;
99 : }
100 :
101 :
102 : /**
103 : * csum_tcpup_magic - Compute an IPv4 pseudo header checksum.
104 : * @saddr: source address
105 : * @daddr: destination address
106 : * @len: length of packet
107 : * @proto: ip protocol of packet
108 : * @sum: initial sum to be added in (32bit unfolded)
109 : *
110 : * Returns the 16bit pseudo header checksum the input data already
111 : * complemented and ready to be filled in.
112 : */
113 459 : static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
114 : __u32 len, __u8 proto,
115 : __wsum sum)
116 : {
117 459 : return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));
118 : }
119 :
120 : /**
121 : * csum_partial - Compute an internet checksum.
122 : * @buff: buffer to be checksummed
123 : * @len: length of buffer.
124 : * @sum: initial sum to be added in (32bit unfolded)
125 : *
126 : * Returns the 32bit unfolded internet checksum of the buffer.
127 : * Before filling it in it needs to be csum_fold()'ed.
128 : * buff should be aligned to a 64bit boundary if possible.
129 : */
130 : extern __wsum csum_partial(const void *buff, int len, __wsum sum);
131 :
132 : /* Do not call this directly. Use the wrappers below */
133 : extern __visible __wsum csum_partial_copy_generic(const void *src, void *dst, int len);
134 :
135 : extern __wsum csum_and_copy_from_user(const void __user *src, void *dst, int len);
136 : extern __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len);
137 : extern __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
138 :
139 : /**
140 : * ip_compute_csum - Compute an 16bit IP checksum.
141 : * @buff: buffer address.
142 : * @len: length of buffer.
143 : *
144 : * Returns the 16bit folded/inverted checksum of the passed buffer.
145 : * Ready to fill in.
146 : */
147 : extern __sum16 ip_compute_csum(const void *buff, int len);
148 :
149 : /**
150 : * csum_ipv6_magic - Compute checksum of an IPv6 pseudo header.
151 : * @saddr: source address
152 : * @daddr: destination address
153 : * @len: length of packet
154 : * @proto: protocol of packet
155 : * @sum: initial sum (32bit unfolded) to be added in
156 : *
157 : * Computes an IPv6 pseudo header checksum. This sum is added the checksum
158 : * into UDP/TCP packets and contains some link layer information.
159 : * Returns the unfolded 32bit checksum.
160 : */
161 :
162 : struct in6_addr;
163 :
164 : #define _HAVE_ARCH_IPV6_CSUM 1
165 : extern __sum16
166 : csum_ipv6_magic(const struct in6_addr *saddr, const struct in6_addr *daddr,
167 : __u32 len, __u8 proto, __wsum sum);
168 :
169 5747 : static inline unsigned add32_with_carry(unsigned a, unsigned b)
170 : {
171 5747 : asm("addl %2,%0\n\t"
172 : "adcl $0,%0"
173 : : "=r" (a)
174 : : "0" (a), "rm" (b));
175 4189 : return a;
176 : }
177 :
178 : #define HAVE_ARCH_CSUM_ADD
179 1073 : static inline __wsum csum_add(__wsum csum, __wsum addend)
180 : {
181 749 : return (__force __wsum)add32_with_carry((__force unsigned)csum,
182 : (__force unsigned)addend);
183 : }
184 :
185 : #endif /* _ASM_X86_CHECKSUM_64_H */
|
