Annotation of embedaddon/rsync/simd-checksum-x86_64.cpp, revision 1.1.1.1
1.1 misho 1: /*
2: * SSE2/SSSE3/AVX2-optimized routines to support checksumming of bytes.
3: *
4: * Copyright (C) 1996 Andrew Tridgell
5: * Copyright (C) 1996 Paul Mackerras
6: * Copyright (C) 2004-2020 Wayne Davison
7: * Copyright (C) 2020 Jorrit Jongma
8: *
9: * This program is free software; you can redistribute it and/or modify
10: * it under the terms of the GNU General Public License as published by
11: * the Free Software Foundation; either version 3 of the License, or
12: * (at your option) any later version.
13: *
14: * This program is distributed in the hope that it will be useful,
15: * but WITHOUT ANY WARRANTY; without even the implied warranty of
16: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17: * GNU General Public License for more details.
18: *
19: * You should have received a copy of the GNU General Public License along
20: * with this program; if not, visit the http://fsf.org website.
21: */
22: /*
23: * Optimization target for get_checksum1() was the Intel Atom D2700, the
24: * slowest CPU in the test set and the most likely to be CPU limited during
25: * transfers. The combination of intrinsics was chosen specifically for the
26: * most gain on that CPU, other combinations were occasionally slightly
27: * faster on the others.
28: *
29: * While on more modern CPUs transfers are less likely to be CPU limited
30: * (at least by this specific function), lower CPU usage is always better.
31: * Improvements may still be seen when matching chunks from NVMe storage
32: * even on newer CPUs.
33: *
34: * Benchmarks (in MB/s) C SSE2 SSSE3 AVX2
35: * - Intel Atom D2700 550 750 1000 N/A
36: * - Intel i7-7700hq 1850 2550 4050 6200
37: * - AMD ThreadRipper 2950x 2900 5600 8950 8100
38: *
39: * Curiously the AMD is slower with AVX2 than SSSE3, while the Intel is
40: * significantly faster. AVX2 is kept because it's more likely to relieve
41: * the bottleneck on the slower CPU.
42: *
43: * This optimization for get_checksum1() is intentionally limited to x86-64
44: * as no 32-bit CPU was available for testing. As 32-bit CPUs only have half
45: * the available xmm registers, this optimized version may not be faster than
46: * the pure C version anyway. Note that all x86-64 CPUs support at least SSE2.
47: *
48: * This file is compiled using GCC 4.8+/clang 6+'s C++ front end to allow the
49: * use of the target attribute, selecting the fastest code path based on
50: * dispatch priority (GCC 5) or runtime detection of CPU capabilities (GCC 6+).
51: * GCC 4.x are not supported to ease configure.ac logic.
52: *
53: * ----
54: *
55: * get_checksum2() is optimized for the case where the selected transfer
56: * checksum is MD5. MD5 can't be made significantly faster with SIMD
57: * instructions than the assembly version already included but SIMD
58: * instructions can be used to hash multiple streams in parallel (see
59: * simd-md5-parallel-x86_64.cpp for details and benchmarks). As rsync's
60: * block-matching algorithm hashes the blocks independently (in contrast to
61: * the whole-file checksum) this method can be employed here.
62: *
63: * To prevent needing to modify the core rsync sources significantly, a
64: * prefetching strategy is used. When a checksum2 is requested, the code
65: * reads ahead several blocks, creates the MD5 hashes for each block in
66: * parallel, returns the hash for the first block, and caches the results
67: * for the other blocks to return in future calls to get_checksum2().
68: */
69:
70: #ifdef __x86_64__
71: #ifdef __cplusplus
72:
73: extern "C" {
74:
75: #include "rsync.h"
76:
77: }
78:
79: #ifdef HAVE_SIMD
80:
81: #include <immintrin.h>
82:
83: /* Some clang versions don't like it when you use static with multi-versioned functions: linker errors */
84: #ifdef __clang__
85: #define MVSTATIC
86: #else
87: #define MVSTATIC static
88: #endif
89:
90: // Missing from the headers on gcc 6 and older, clang 8 and older
91: typedef long long __m128i_u __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));
92: typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, __aligned__(1)));
93:
94: /* Compatibility macros to let our SSSE3 algorithm run with only SSE2.
95: These used to be neat individual functions with target attributes switching between SSE2 and SSSE3 implementations
96: as needed, but though this works perfectly with GCC, clang fails to inline those properly leading to a near 50%
97: performance drop - combined with static and inline modifiers gets you linker errors and even compiler crashes...
98: */
99:
100: #define SSE2_INTERLEAVE_ODD_EPI16(a, b) _mm_packs_epi32(_mm_srai_epi32(a, 16), _mm_srai_epi32(b, 16))
101: #define SSE2_INTERLEAVE_EVEN_EPI16(a, b) SSE2_INTERLEAVE_ODD_EPI16(_mm_slli_si128(a, 2), _mm_slli_si128(b, 2))
102: #define SSE2_MULU_ODD_EPI8(a, b) _mm_mullo_epi16(_mm_srli_epi16(a, 8), _mm_srai_epi16(b, 8))
103: #define SSE2_MULU_EVEN_EPI8(a, b) _mm_mullo_epi16(_mm_and_si128(a, _mm_set1_epi16(0xFF)), _mm_srai_epi16(_mm_slli_si128(b, 1), 8))
104:
105: #define SSE2_HADDS_EPI16(a, b) _mm_adds_epi16(SSE2_INTERLEAVE_EVEN_EPI16(a, b), SSE2_INTERLEAVE_ODD_EPI16(a, b))
106: #define SSE2_MADDUBS_EPI16(a, b) _mm_adds_epi16(SSE2_MULU_EVEN_EPI8(a, b), SSE2_MULU_ODD_EPI8(a, b))
107:
108: __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
109: __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
110: __attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; }
111:
112: /*
113: Original loop per 4 bytes:
114: s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3] + 10*CHAR_OFFSET;
115: s1 += buf[i] + buf[i+1] + buf[i+2] + buf[i+3] + 4*CHAR_OFFSET;
116:
117: SSE2/SSSE3 loop per 32 bytes:
118: int16 t1[8];
119: int16 t2[8];
120: for (int j = 0; j < 8; j++) {
121: t1[j] = buf[j*4 + i] + buf[j*4 + i+1] + buf[j*4 + i+2] + buf[j*4 + i+3];
122: t2[j] = 4*buf[j*4 + i] + 3*buf[j*4 + i+1] + 2*buf[j*4 + i+2] + buf[j*4 + i+3];
123: }
124: s2 += 32*s1 + (uint32)(
125: 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6] +
126: t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7]
127: ) + 528*CHAR_OFFSET;
128: s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]) +
129: 32*CHAR_OFFSET;
130: */
131: __attribute__ ((target("ssse3"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
132: {
133: if (len > 32) {
134: int aligned = ((uintptr_t)buf & 15) == 0;
135:
136: uint32 x[4] = {0};
137: x[0] = *ps1;
138: __m128i ss1 = _mm_loadu_si128((__m128i_u*)x);
139: x[0] = *ps2;
140: __m128i ss2 = _mm_loadu_si128((__m128i_u*)x);
141:
142: const int16 mul_t1_buf[8] = {28, 24, 20, 16, 12, 8, 4, 0};
143: __m128i mul_t1 = _mm_loadu_si128((__m128i_u*)mul_t1_buf);
144:
145: for (; i < (len-32); i+=32) {
146: // Load ... 2*[int8*16]
147: __m128i in8_1, in8_2;
148: if (!aligned) {
149: // Synonymous with _mm_loadu_si128 on all but a handful of old CPUs
150: in8_1 = _mm_lddqu_si128((__m128i_u*)&buf[i]);
151: in8_2 = _mm_lddqu_si128((__m128i_u*)&buf[i + 16]);
152: } else {
153: in8_1 = _mm_load_si128((__m128i_u*)&buf[i]);
154: in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]);
155: }
156:
157: // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
158: // Fastest, even though multiply by 1
159: __m128i mul_one = _mm_set1_epi8(1);
160: __m128i add16_1 = _mm_maddubs_epi16(mul_one, in8_1);
161: __m128i add16_2 = _mm_maddubs_epi16(mul_one, in8_2);
162:
163: // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
164: __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
165: __m128i mul_add16_1 = _mm_maddubs_epi16(mul_const, in8_1);
166: __m128i mul_add16_2 = _mm_maddubs_epi16(mul_const, in8_2);
167:
168: // s2 += 32*s1
169: ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5));
170:
171: // [sum(t1[0]..t1[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
172: // Shifting left, then shifting right again and shuffling (rather than just
173: // shifting right as with mul32 below) to cheaply end up with the correct sign
174: // extension as we go from int16 to int32.
175: __m128i sum_add32 = _mm_add_epi16(add16_1, add16_2);
176: sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 2));
177: sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 4));
178: sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 8));
179: sum_add32 = _mm_srai_epi32(sum_add32, 16);
180: sum_add32 = _mm_shuffle_epi32(sum_add32, 3);
181:
182: // [sum(t2[0]..t2[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
183: __m128i sum_mul_add32 = _mm_add_epi16(mul_add16_1, mul_add16_2);
184: sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 2));
185: sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 4));
186: sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 8));
187: sum_mul_add32 = _mm_srai_epi32(sum_mul_add32, 16);
188: sum_mul_add32 = _mm_shuffle_epi32(sum_mul_add32, 3);
189:
190: // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]
191: ss1 = _mm_add_epi32(ss1, sum_add32);
192:
193: // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7]
194: ss2 = _mm_add_epi32(ss2, sum_mul_add32);
195:
196: // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
197: // We could've combined this with generating sum_add32 above and
198: // save an instruction but benchmarking shows that as being slower
199: __m128i add16 = _mm_hadds_epi16(add16_1, add16_2);
200:
201: // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
202: __m128i mul32 = _mm_madd_epi16(add16, mul_t1);
203:
204: // [sum(mul32), X, X, X] [int32*4]; faster than multiple _mm_hadd_epi32
205: mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 4));
206: mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 8));
207:
208: // s2 += 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6]
209: ss2 = _mm_add_epi32(ss2, mul32);
210:
211: #if CHAR_OFFSET != 0
212: // s1 += 32*CHAR_OFFSET
213: __m128i char_offset_multiplier = _mm_set1_epi32(32 * CHAR_OFFSET);
214: ss1 = _mm_add_epi32(ss1, char_offset_multiplier);
215:
216: // s2 += 528*CHAR_OFFSET
217: char_offset_multiplier = _mm_set1_epi32(528 * CHAR_OFFSET);
218: ss2 = _mm_add_epi32(ss2, char_offset_multiplier);
219: #endif
220: }
221:
222: _mm_store_si128((__m128i_u*)x, ss1);
223: *ps1 = x[0];
224: _mm_store_si128((__m128i_u*)x, ss2);
225: *ps2 = x[0];
226: }
227: return i;
228: }
229:
230: /*
231: Same as SSSE3 version, but using macros defined above to emulate SSSE3 calls that are not available with SSE2.
232: For GCC-only the SSE2 and SSSE3 versions could be a single function calling other functions with the right
233: target attributes to emulate SSSE3 calls on SSE2 if needed, but clang doesn't inline those properly leading
234: to a near 50% performance drop.
235: */
236: __attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
237: {
238: if (len > 32) {
239: int aligned = ((uintptr_t)buf & 15) == 0;
240:
241: uint32 x[4] = {0};
242: x[0] = *ps1;
243: __m128i ss1 = _mm_loadu_si128((__m128i_u*)x);
244: x[0] = *ps2;
245: __m128i ss2 = _mm_loadu_si128((__m128i_u*)x);
246:
247: const int16 mul_t1_buf[8] = {28, 24, 20, 16, 12, 8, 4, 0};
248: __m128i mul_t1 = _mm_loadu_si128((__m128i_u*)mul_t1_buf);
249:
250: for (; i < (len-32); i+=32) {
251: // Load ... 2*[int8*16]
252: __m128i in8_1, in8_2;
253: if (!aligned) {
254: in8_1 = _mm_loadu_si128((__m128i_u*)&buf[i]);
255: in8_2 = _mm_loadu_si128((__m128i_u*)&buf[i + 16]);
256: } else {
257: in8_1 = _mm_load_si128((__m128i_u*)&buf[i]);
258: in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]);
259: }
260:
261: // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8]
262: // Fastest, even though multiply by 1
263: __m128i mul_one = _mm_set1_epi8(1);
264: __m128i add16_1 = SSE2_MADDUBS_EPI16(mul_one, in8_1);
265: __m128i add16_2 = SSE2_MADDUBS_EPI16(mul_one, in8_2);
266:
267: // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8]
268: __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
269: __m128i mul_add16_1 = SSE2_MADDUBS_EPI16(mul_const, in8_1);
270: __m128i mul_add16_2 = SSE2_MADDUBS_EPI16(mul_const, in8_2);
271:
272: // s2 += 32*s1
273: ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5));
274:
275: // [sum(t1[0]..t1[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
276: // Shifting left, then shifting right again and shuffling (rather than just
277: // shifting right as with mul32 below) to cheaply end up with the correct sign
278: // extension as we go from int16 to int32.
279: __m128i sum_add32 = _mm_add_epi16(add16_1, add16_2);
280: sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 2));
281: sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 4));
282: sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 8));
283: sum_add32 = _mm_srai_epi32(sum_add32, 16);
284: sum_add32 = _mm_shuffle_epi32(sum_add32, 3);
285:
286: // [sum(t2[0]..t2[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16
287: __m128i sum_mul_add32 = _mm_add_epi16(mul_add16_1, mul_add16_2);
288: sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 2));
289: sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 4));
290: sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 8));
291: sum_mul_add32 = _mm_srai_epi32(sum_mul_add32, 16);
292: sum_mul_add32 = _mm_shuffle_epi32(sum_mul_add32, 3);
293:
294: // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]
295: ss1 = _mm_add_epi32(ss1, sum_add32);
296:
297: // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7]
298: ss2 = _mm_add_epi32(ss2, sum_mul_add32);
299:
300: // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8]
301: // We could've combined this with generating sum_add32 above and
302: // save an instruction but benchmarking shows that as being slower
303: __m128i add16 = SSE2_HADDS_EPI16(add16_1, add16_2);
304:
305: // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4]
306: __m128i mul32 = _mm_madd_epi16(add16, mul_t1);
307:
308: // [sum(mul32), X, X, X] [int32*4]; faster than multiple _mm_hadd_epi32
309: mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 4));
310: mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 8));
311:
312: // s2 += 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6]
313: ss2 = _mm_add_epi32(ss2, mul32);
314:
315: #if CHAR_OFFSET != 0
316: // s1 += 32*CHAR_OFFSET
317: __m128i char_offset_multiplier = _mm_set1_epi32(32 * CHAR_OFFSET);
318: ss1 = _mm_add_epi32(ss1, char_offset_multiplier);
319:
320: // s2 += 528*CHAR_OFFSET
321: char_offset_multiplier = _mm_set1_epi32(528 * CHAR_OFFSET);
322: ss2 = _mm_add_epi32(ss2, char_offset_multiplier);
323: #endif
324: }
325:
326: _mm_store_si128((__m128i_u*)x, ss1);
327: *ps1 = x[0];
328: _mm_store_si128((__m128i_u*)x, ss2);
329: *ps2 = x[0];
330: }
331: return i;
332: }
333:
334: /*
335: AVX2 loop per 64 bytes:
336: int16 t1[16];
337: int16 t2[16];
338: for (int j = 0; j < 16; j++) {
339: t1[j] = buf[j*4 + i] + buf[j*4 + i+1] + buf[j*4 + i+2] + buf[j*4 + i+3];
340: t2[j] = 4*buf[j*4 + i] + 3*buf[j*4 + i+1] + 2*buf[j*4 + i+2] + buf[j*4 + i+3];
341: }
342: s2 += 64*s1 + (uint32)(
343: 60*t1[0] + 56*t1[1] + 52*t1[2] + 48*t1[3] + 44*t1[4] + 40*t1[5] + 36*t1[6] + 32*t1[7] + 28*t1[8] + 24*t1[9] + 20*t1[10] + 16*t1[11] + 12*t1[12] + 8*t1[13] + 4*t1[14] +
344: t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + t2[8] + t2[9] + t2[10] + t2[11] + t2[12] + t2[13] + t2[14] + t2[15]
345: ) + 2080*CHAR_OFFSET;
346: s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]) +
347: 64*CHAR_OFFSET;
348: */
349: __attribute__ ((target("avx2"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
350: {
351: if (len > 64) {
352: // Instructions reshuffled compared to SSE2 for slightly better performance
353: int aligned = ((uintptr_t)buf & 31) == 0;
354:
355: uint32 x[8] = {0};
356: x[0] = *ps1;
357: __m256i ss1 = _mm256_lddqu_si256((__m256i_u*)x);
358: x[0] = *ps2;
359: __m256i ss2 = _mm256_lddqu_si256((__m256i_u*)x);
360:
361: // The order gets shuffled compared to SSE2
362: const int16 mul_t1_buf[16] = {60, 56, 52, 48, 28, 24, 20, 16, 44, 40, 36, 32, 12, 8, 4, 0};
363: __m256i mul_t1 = _mm256_lddqu_si256((__m256i_u*)mul_t1_buf);
364:
365: for (; i < (len-64); i+=64) {
366: // Load ... 2*[int8*32]
367: __m256i in8_1, in8_2;
368: if (!aligned) {
369: in8_1 = _mm256_lddqu_si256((__m256i_u*)&buf[i]);
370: in8_2 = _mm256_lddqu_si256((__m256i_u*)&buf[i + 32]);
371: } else {
372: in8_1 = _mm256_load_si256((__m256i_u*)&buf[i]);
373: in8_2 = _mm256_load_si256((__m256i_u*)&buf[i + 32]);
374: }
375:
376: // Prefetch for next loops. This has no observable effect on the
377: // tested AMD but makes as much as 20% difference on the Intel.
378: // Curiously that same Intel sees no benefit from this with SSE2
379: // or SSSE3.
380: _mm_prefetch(&buf[i + 64], _MM_HINT_T0);
381: _mm_prefetch(&buf[i + 96], _MM_HINT_T0);
382: _mm_prefetch(&buf[i + 128], _MM_HINT_T0);
383: _mm_prefetch(&buf[i + 160], _MM_HINT_T0);
384:
385: // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*16]
386: // Fastest, even though multiply by 1
387: __m256i mul_one = _mm256_set1_epi8(1);
388: __m256i add16_1 = _mm256_maddubs_epi16(mul_one, in8_1);
389: __m256i add16_2 = _mm256_maddubs_epi16(mul_one, in8_2);
390:
391: // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*16]
392: __m256i mul_const = _mm256_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24));
393: __m256i mul_add16_1 = _mm256_maddubs_epi16(mul_const, in8_1);
394: __m256i mul_add16_2 = _mm256_maddubs_epi16(mul_const, in8_2);
395:
396: // s2 += 64*s1
397: ss2 = _mm256_add_epi32(ss2, _mm256_slli_epi32(ss1, 6));
398:
399: // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*16]
400: __m256i add16 = _mm256_hadds_epi16(add16_1, add16_2);
401:
402: // [t1[0], t1[1], ...] -> [t1[0]*60 + t1[1]*56, ...] [int32*8]
403: __m256i mul32 = _mm256_madd_epi16(add16, mul_t1);
404:
405: // [sum(t1[0]..t1[15]), X, X, X, X, X, X, X] [int32*8]
406: __m256i sum_add32 = _mm256_add_epi16(add16_1, add16_2);
407: sum_add32 = _mm256_add_epi16(sum_add32, _mm256_permute4x64_epi64(sum_add32, 2 + (3 << 2) + (0 << 4) + (1 << 6)));
408: sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 2));
409: sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 4));
410: sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 8));
411: sum_add32 = _mm256_srai_epi32(sum_add32, 16);
412: sum_add32 = _mm256_shuffle_epi32(sum_add32, 3);
413:
414: // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]
415: ss1 = _mm256_add_epi32(ss1, sum_add32);
416:
417: // [sum(t2[0]..t2[15]), X, X, X, X, X, X, X] [int32*8]
418: __m256i sum_mul_add32 = _mm256_add_epi16(mul_add16_1, mul_add16_2);
419: sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_permute4x64_epi64(sum_mul_add32, 2 + (3 << 2) + (0 << 4) + (1 << 6)));
420: sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 2));
421: sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 4));
422: sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 8));
423: sum_mul_add32 = _mm256_srai_epi32(sum_mul_add32, 16);
424: sum_mul_add32 = _mm256_shuffle_epi32(sum_mul_add32, 3);
425:
426: // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + t2[8] + t2[9] + t2[10] + t2[11] + t2[12] + t2[13] + t2[14] + t2[15]
427: ss2 = _mm256_add_epi32(ss2, sum_mul_add32);
428:
429: // [sum(mul32), X, X, X, X, X, X, X] [int32*8]
430: mul32 = _mm256_add_epi32(mul32, _mm256_permute2x128_si256(mul32, mul32, 1));
431: mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 4));
432: mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 8));
433:
434: // s2 += 60*t1[0] + 56*t1[1] + 52*t1[2] + 48*t1[3] + 44*t1[4] + 40*t1[5] + 36*t1[6] + 32*t1[7] + 28*t1[8] + 24*t1[9] + 20*t1[10] + 16*t1[11] + 12*t1[12] + 8*t1[13] + 4*t1[14]
435: ss2 = _mm256_add_epi32(ss2, mul32);
436:
437: #if CHAR_OFFSET != 0
438: // s1 += 64*CHAR_OFFSET
439: __m256i char_offset_multiplier = _mm256_set1_epi32(64 * CHAR_OFFSET);
440: ss1 = _mm256_add_epi32(ss1, char_offset_multiplier);
441:
442: // s2 += 2080*CHAR_OFFSET
443: char_offset_multiplier = _mm256_set1_epi32(2080 * CHAR_OFFSET);
444: ss2 = _mm256_add_epi32(ss2, char_offset_multiplier);
445: #endif
446: }
447:
448: _mm256_store_si256((__m256i_u*)x, ss1);
449: *ps1 = x[0];
450: _mm256_store_si256((__m256i_u*)x, ss2);
451: *ps2 = x[0];
452: }
453: return i;
454: }
455:
456: static int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2)
457: {
458: uint32 s1 = *ps1;
459: uint32 s2 = *ps2;
460: for (; i < (len-4); i+=4) {
461: s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3] + 10*CHAR_OFFSET;
462: s1 += (buf[i+0] + buf[i+1] + buf[i+2] + buf[i+3] + 4*CHAR_OFFSET);
463: }
464: for (; i < len; i++) {
465: s1 += (buf[i]+CHAR_OFFSET); s2 += s1;
466: }
467: *ps1 = s1;
468: *ps2 = s2;
469: return i;
470: }
471:
472: /* With GCC 10 putting this implementation inside 'extern "C"' causes an
473: assembler error. That worked fine on GCC 5-9 and clang 6-10...
474: */
475: static inline uint32 get_checksum1_cpp(char *buf1, int32 len)
476: {
477: int32 i = 0;
478: uint32 s1 = 0;
479: uint32 s2 = 0;
480:
481: // multiples of 64 bytes using AVX2 (if available)
482: i = get_checksum1_avx2_64((schar*)buf1, len, i, &s1, &s2);
483:
484: // multiples of 32 bytes using SSSE3 (if available)
485: i = get_checksum1_ssse3_32((schar*)buf1, len, i, &s1, &s2);
486:
487: // multiples of 32 bytes using SSE2 (if available)
488: i = get_checksum1_sse2_32((schar*)buf1, len, i, &s1, &s2);
489:
490: // whatever is left
491: i = get_checksum1_default_1((schar*)buf1, len, i, &s1, &s2);
492:
493: return (s1 & 0xffff) + (s2 << 16);
494: }
495:
496: extern "C" {
497:
498: uint32 get_checksum1(char *buf1, int32 len)
499: {
500: return get_checksum1_cpp(buf1, len);
501: }
502:
503: #if !defined(BENCHMARK_SIMD_CHECKSUM1)
504:
505: // see simd-md5-parallel-x86_64.cpp
506: extern int md5_parallel_slots();
507: extern int md5_parallel(int streams, char** buf, int* len, char** sum, char* pre4, char* post4);
508:
509: #endif /* !BENCHMARK_SIMD_CHECKSUM1 */
510:
511: #if !defined(BENCHMARK_SIMD_CHECKSUM1) && !defined(BENCHMARK_SIMD_CHECKSUM2)
512:
513: #define PREFETCH_ENABLE 1 // debugging
514:
515: #if 0 // debugging
516: #define PREFETCH_PRINTF(f_, ...) printf((f_), ##__VA_ARGS__)
517: #else
518: #define PREFETCH_PRINTF(f_, ...) (void)0;
519: #endif
520:
521: #define PREFETCH_MIN_LEN 1024 // the overhead is unlikely to be worth the gain for small blocks
522: #define PREFETCH_MAX_BLOCKS 8
523:
524: typedef struct {
525: int in_use;
526: OFF_T offset;
527: int32 len;
528: char sum[SUM_LENGTH];
529: } prefetch_sum_t;
530:
531: typedef struct {
532: struct map_struct *map;
533: OFF_T len;
534: OFF_T last;
535: int32 blocklen;
536: int blocks;
537: prefetch_sum_t sums[PREFETCH_MAX_BLOCKS];
538: } prefetch_t;
539:
540: prefetch_t *prefetch;
541:
542: extern int xfersum_type;
543: extern int checksum_seed;
544: extern int proper_seed_order;
545: extern void get_checksum2_nosimd(char *buf, int32 len, char *sum, OFF_T prefetch_offset);
546:
547: extern char *map_ptr(struct map_struct *map, OFF_T offset, int32 len);
548:
549: void checksum2_disable_prefetch()
550: {
551: if (prefetch) {
552: PREFETCH_PRINTF("checksum2_disable_prefetch\n");
553: free(prefetch);
554: prefetch = NULL;
555: }
556: }
557:
558: void checksum2_enable_prefetch(UNUSED(struct map_struct *map), UNUSED(OFF_T len), UNUSED(int32 blocklen))
559: {
560: #ifdef PREFETCH_ENABLE
561: checksum2_disable_prefetch();
562: int slots = md5_parallel_slots();
563: if ((xfersum_type == CSUM_MD5 || xfersum_type == CSUM_MD5P8) && slots > 1 && len >= blocklen * PREFETCH_MAX_BLOCKS && blocklen >= PREFETCH_MIN_LEN) {
564: prefetch = (prefetch_t*)malloc(sizeof(prefetch_t));
565: memset(prefetch, 0, sizeof(prefetch_t));
566: prefetch->map = map;
567: prefetch->len = len;
568: prefetch->last = 0;
569: prefetch->blocklen = blocklen;
570: prefetch->blocks = MIN(PREFETCH_MAX_BLOCKS, slots);
571: PREFETCH_PRINTF("checksum2_enable_prefetch len:%ld blocklen:%d blocks:%d\n", prefetch->len, prefetch->blocklen, prefetch->blocks);
572: }
573: #endif
574: }
575:
576: static inline void checksum2_reset_prefetch()
577: {
578: for (int i = 0; i < PREFETCH_MAX_BLOCKS; i++) {
579: prefetch->sums[i].in_use = 0;
580: }
581: }
582:
583: static int get_checksum2_prefetched(int32 len, char* sum, OFF_T prefetch_offset)
584: {
585: if (prefetch->sums[0].in_use) {
586: if ((prefetch->sums[0].offset == prefetch_offset) && (prefetch->sums[0].len == len)) {
587: memcpy(sum, prefetch->sums[0].sum, SUM_LENGTH);
588: for (int i = 0; i < PREFETCH_MAX_BLOCKS - 1; i++) {
589: prefetch->sums[i] = prefetch->sums[i + 1];
590: }
591: prefetch->sums[PREFETCH_MAX_BLOCKS - 1].in_use = 0;
592: PREFETCH_PRINTF("checksum2_prefetch HIT len:%d offset:%ld\n", len, prefetch_offset);
593: return 1;
594: } else {
595: // unexpected access, reset cache
596: PREFETCH_PRINTF("checksum2_prefetch MISS len:%d offset:%ld\n", len, prefetch_offset);
597: checksum2_reset_prefetch();
598: }
599: }
600: return 0;
601: }
602:
603: static int checksum2_perform_prefetch(OFF_T prefetch_offset)
604: {
605: int blocks = MIN(MAX(1, (prefetch->len + prefetch->blocklen - 1) / prefetch->blocklen), prefetch->blocks);
606: if (blocks < 2) return 0; // fall through to non-simd, probably faster
607:
608: int32 total = 0;
609: int i;
610: for (i = 0; i < blocks; i++) {
611: prefetch->sums[i].offset = prefetch_offset + total;
612: prefetch->sums[i].len = MIN(prefetch->blocklen, prefetch->len - prefetch_offset - total);
613: prefetch->sums[i].in_use = 0;
614: total += prefetch->sums[i].len;
615: }
616: for (; i < PREFETCH_MAX_BLOCKS; i++) {
617: prefetch->sums[i].in_use = 0;
618: }
619:
620: uchar seedbuf[4];
621: SIVALu(seedbuf, 0, checksum_seed);
622:
623: PREFETCH_PRINTF("checksum2_perform_prefetch pos:%ld len:%d blocks:%d\n", prefetch_offset, total, blocks);
624: char* mapbuf = map_ptr(prefetch->map, prefetch_offset, total);
625: char* bufs[PREFETCH_MAX_BLOCKS] = {0};
626: int lens[PREFETCH_MAX_BLOCKS] = {0};
627: char* sums[PREFETCH_MAX_BLOCKS] = {0};
628: for (i = 0; i < blocks; i++) {
629: bufs[i] = mapbuf + prefetch->sums[i].offset - prefetch_offset;
630: lens[i] = prefetch->sums[i].len;
631: sums[i] = prefetch->sums[i].sum;
632: }
633: if (md5_parallel(blocks, bufs, lens, sums, (proper_seed_order && checksum_seed) ? (char*)seedbuf : NULL, (!proper_seed_order && checksum_seed) ? (char*)seedbuf : NULL)) {
634: for (i = 0; i < blocks; i++) {
635: prefetch->sums[i].in_use = 1;
636: }
637: return 1;
638: } else {
639: // this should never be, abort
640: PREFETCH_PRINTF("checksum2_perform_prefetch PMD5 ABORT\n");
641: checksum2_disable_prefetch();
642: }
643: return 0;
644: }
645:
646: void get_checksum2(char *buf, int32 len, char *sum, OFF_T prefetch_offset)
647: {
648: if (prefetch) {
649: PREFETCH_PRINTF("get_checksum2 %d @ %ld\n", len, prefetch_offset);
650: OFF_T last = prefetch->last;
651: prefetch->last = prefetch_offset;
652: if ((prefetch_offset != 0) && (prefetch_offset != last + prefetch->blocklen)) {
653: // we're looking around trying to align blocks, prefetching will slow things down
654: PREFETCH_PRINTF("get_checksum2 SEEK\n");
655: checksum2_reset_prefetch();
656: } else if (get_checksum2_prefetched(len, sum, prefetch_offset)) {
657: // hit
658: return;
659: } else if (checksum2_perform_prefetch(prefetch_offset)) {
660: if (get_checksum2_prefetched(len, sum, prefetch_offset)) {
661: // hit; should always be as we just fetched this data
662: return;
663: } else {
664: // this should never be, abort
665: PREFETCH_PRINTF("get_checksum2 MISSING DATA ABORT\n");
666: checksum2_disable_prefetch();
667: }
668: }
669: }
670: get_checksum2_nosimd(buf, len, sum, prefetch_offset);
671: }
672: #endif /* !BENCHMARK_SIMD_CHECKSUM1 && !BENCHMARK_SIMD_CHECKSUM2 */
673:
674: } // "C"
675:
676: /* Benchmark compilation
677:
678: The get_checksum1() benchmark runs through all available code paths in a
679: single execution, the get_checksum2()/MD5 and MD5P8 benchmark needs to be
680: recompiled for each code path (it always uses the fastest path available
681: on the current CPU otherwise). Note that SSE2/AVX2 MD5 optimizations will
682: be used when applicable regardless of rsync being built with OpenSSL.
683:
684: Something like the following should compile and run the benchmarks:
685:
686: # if gcc
687: export CC=gcc
688: export CXX=g++
689: export CXX_BASE="-g -O3 -fno-exceptions -fno-rtti"
690:
691: # else if clang
692: export CC=clang
693: export CXX=clang++
694: export CXX_BASE="-g -O3 -fno-exceptions -fno-rtti -fno-slp-vectorize"
695:
696: # /if
697:
698: export CONF_EXTRA="--disable-md2man --disable-zstd --disable-lz4 --disable-xxhash"
699: export CXX_CSUM1="$CXX_BASE simd-checksum-x86_64.cpp"
700: export CXX_MD5P="$CXX_BASE -c -o simd-md5-parallel-x86_64.o simd-md5-parallel-x86_64.cpp"
701: export CXX_CSUM2="$CXX_BASE simd-checksum-x86_64.cpp simd-md5-parallel-x86_64.o lib/md5.o lib/md5p8.o lib/md5-asm-x86_64.o"
702:
703: rm bench_csum*
704:
705: ./configure --disable-openssl --enable-simd $CONF_EXTRA && make clean && make -j4
706:
707: $CXX -DBENCHMARK_SIMD_CHECKSUM1 $CXX_CSUM1 -o bench_csum1.all
708:
709: $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_MD5P
710: $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_CSUM2 -o bench_csum2.asm
711:
712: $CXX -DBENCHMARK_SIMD_CHECKSUM2 -DPMD5_ALLOW_SSE2 $CXX_MD5P
713: $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_CSUM2 -o bench_csum2.sse2
714:
715: $CXX -DBENCHMARK_SIMD_CHECKSUM2 -DPMD5_ALLOW_AVX2 $CXX_MD5P
716: $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_CSUM2 -o bench_csum2.avx2
717:
718: ./configure --enable-openssl --enable-simd $CONF_EXTRA && make clean && make -j4
719:
720: $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_MD5P
721: $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_CSUM2 -o bench_csum2.openssl -lcrypto
722:
723: ./bench_csum1.all
724: ./bench_csum2.asm
725: ./bench_csum2.openssl
726: ./bench_csum2.sse2
727: ./bench_csum2.avx2
728:
729: */
730:
731: #if defined(BENCHMARK_SIMD_CHECKSUM1) || defined(BENCHMARK_SIMD_CHECKSUM2)
732: #pragma clang optimize off
733: #pragma GCC push_options
734: #pragma GCC optimize ("O0")
735:
736: #define ROUNDS 1024
737: #define BLOCK_LEN 1024*1024
738:
739: #ifndef CLOCK_MONOTONIC_RAW
740: #define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC
741: #endif
742: #endif /* BENCHMARK_SIMD_CHECKSUM1 || BENCHMARK_SIMD_CHECKSUM2 */
743:
744: #ifdef BENCHMARK_SIMD_CHECKSUM1
745: static void benchmark(const char* desc, int32 (*func)(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2), schar* buf, int32 len) {
746: struct timespec start, end;
747: uint64_t us;
748: uint32_t cs, s1, s2;
749: int i, next;
750:
751: clock_gettime(CLOCK_MONOTONIC_RAW, &start);
752: for (i = 0; i < ROUNDS; i++) {
753: s1 = s2 = 0;
754: next = func((schar*)buf, len, 0, &s1, &s2);
755: get_checksum1_default_1((schar*)buf, len, next, &s1, &s2);
756: }
757: clock_gettime(CLOCK_MONOTONIC_RAW, &end);
758: us = next == 0 ? 0 : (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_nsec - start.tv_nsec) / 1000;
759: cs = next == 0 ? 0 : (s1 & 0xffff) + (s2 << 16);
760: printf("CSUM1 :: %-5s :: %5.0f MB/s :: %08x\n", desc, us ? (float)(len / (1024 * 1024) * ROUNDS) / ((float)us / 1000000.0f) : 0, cs);
761: }
762:
763: static int32 get_checksum1_auto(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) {
764: uint32 cs = get_checksum1((char*)buf, len);
765: *ps1 = cs & 0xffff;
766: *ps2 = cs >> 16;
767: return len;
768: }
769:
770: int main() {
771: int i;
772: unsigned char* buf = (unsigned char*)malloc(BLOCK_LEN);
773: for (i = 0; i < BLOCK_LEN; i++) buf[i] = (i + (i % 3) + (i % 11)) % 256;
774:
775: benchmark("Auto", get_checksum1_auto, (schar*)buf, BLOCK_LEN);
776: benchmark("Raw-C", get_checksum1_default_1, (schar*)buf, BLOCK_LEN);
777: benchmark("SSE2", get_checksum1_sse2_32, (schar*)buf, BLOCK_LEN);
778: benchmark("SSSE3", get_checksum1_ssse3_32, (schar*)buf, BLOCK_LEN);
779: benchmark("AVX2", get_checksum1_avx2_64, (schar*)buf, BLOCK_LEN);
780:
781: free(buf);
782: return 0;
783: }
784: #endif /* BENCHMARK_SIMD_CHECKSUM1 */
785:
786: #ifdef BENCHMARK_SIMD_CHECKSUM2
787: static void benchmark(const char* desc, void (*func)(char* buf, int32 len, char* sum_out), void (*func2)(char* buf, int32 len, char* sum_out), char* buf, int32 len, int streams) {
788: struct timespec start, end;
789: uint64_t us;
790: unsigned char cs1[16];
791: unsigned char cs2[16];
792: int i;
793:
794: clock_gettime(CLOCK_MONOTONIC_RAW, &start);
795: for (i = 0; i < ROUNDS; i++) {
796: func(buf, len, (char*)cs1);
797: }
798: clock_gettime(CLOCK_MONOTONIC_RAW, &end);
799: us = (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_nsec - start.tv_nsec) / 1000;
800:
801: func2(buf, len, (char*)cs2);
802:
803: float perf = us ? (float)(len / (1024 * 1024) * ROUNDS) / ((float)us / 1000000.0f) : 0;
804: printf("CSUM2 :: %-7s :: %5.0f to %5.0f MB/s :: ", desc, perf, perf * streams);
805: for (i = 0; i < 16; i++) {
806: printf("%02x", cs1[i] & 0xFF);
807: }
808: printf(" :: ");
809: for (i = 0; i < 16; i++) {
810: printf("%02x", cs2[i] & 0xFF);
811: }
812: printf("\n");
813: }
814:
815: static void benchmark_inner(char* buf, int32 len, char* sum_out) {
816: // This should produce the same output for different optimizations
817: // levels, not the same as sanity_check()
818:
819: char* bufs[8] = {0};
820: int lens[8] = {0};
821: char* sums[8] = {0};
822:
823: bufs[0] = buf;
824: lens[0] = len;
825: sums[0] = sum_out;
826: md5_parallel(1, bufs, lens, sums, NULL, NULL);
827: }
828:
829: extern "C" {
830: extern void MD5P8_Init_c(MD5P8_CTX *ctx);
831: extern void MD5P8_Update_c(MD5P8_CTX *ctx, const uchar *input, uint32 length);
832: extern void MD5P8_Final_c(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx);
833: }
834:
835: static void sanity_check(char* buf, int32 len, char* sum_out) {
836: // This should produce the same output for different optimizations
837: // levels, not the same as benchmark_inner()
838: if (md5_parallel_slots() <= 1) {
839: MD5P8_CTX m5p8;
840: MD5P8_Init_c(&m5p8);
841: MD5P8_Update_c(&m5p8, (uchar *)buf, len);
842: MD5P8_Final_c((uchar *)sum_out, &m5p8);
843: } else {
844: MD5P8_CTX m5p8;
845: MD5P8_Init(&m5p8);
846: MD5P8_Update(&m5p8, (uchar *)buf, len);
847: MD5P8_Final((uchar *)sum_out, &m5p8);
848: }
849: }
850:
851: int main() {
852: // This benchmarks the parallel MD5 checksum rather than get_checksum2()
853: // as the latter would require compiling in a lot of rsync's code, but
854: // it touches all the same internals so the performance should be nearly
855: // identical.
856:
857: int i;
858: char* buf = (char*)malloc(BLOCK_LEN);
859: for (i = 0; i < BLOCK_LEN; i++) buf[i] = (i + (i % 3) + (i % 11)) % 256;
860:
861: const char* method = "?";
862: switch (md5_parallel_slots()) {
863: case 8: method = "AVX2"; break;
864: case 4: method = "SSE2"; break;
865: #ifdef USE_OPENSSL
866: case 1: method = "OpenSSL"; break;
867: #elif (CSUM_CHUNK == 64)
868: case 1: method = "ASM"; break;
869: #else
870: // this won't happen unless you modified code somewhere
871: case 1: method = "Raw-C"; break;
872: #endif
873: }
874:
875: benchmark(method, benchmark_inner, sanity_check, buf, BLOCK_LEN, md5_parallel_slots());
876:
877: free(buf);
878: return 0;
879: }
880: #endif /* BENCHMARK_SIMD_CHECKSUM2 */
881:
882: #if defined(BENCHMARK_SIMD_CHECKSUM1) || defined(BENCHMARK_SIMD_CHECKSUM2)
883: #pragma GCC pop_options
884: #pragma clang optimize on
885: #endif /* BENCHMARK_SIMD_CHECKSUM1 || BENCHMARK_SIMD_CHECKSUM2 */
886:
887: #endif /* HAVE_SIMD */
888: #endif /* __cplusplus */
889: #endif /* __x86_64__ */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to simd-checksum-x86_64.cpp CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / rsync