Annotation of embedaddon/rsync/patches/md5p8.diff, revision 1.1
1.1 ! misho 1: From: Jorrit Jongma <git@jongma.org>
! 2:
! 3: - MD5 optimization in block matching phase:
! 4:
! 5: MD5 hashes computed during rsync's block matching phase are independent
! 6: and thus possible to process in parallel. This code processes 4 blocks
! 7: in parallel if SSE2 is available, or 8 if AVX2 is available. An increase
! 8: of performance (or decrease of CPU usage) of up to 6x has been measured.
! 9:
! 10: A prefetching algorithm is used to predict and load upcoming blocks, as
! 11: this prevents the need for extensive modifications to other parts of
! 12: the rsync sources to get this working.
! 13:
! 14: This remains compatible with existing rsync builds using MD5 checksums.
! 15:
! 16: - MD5P8 whole-file checksum:
! 17:
! 18: Splits the input up into 8 independent streams (64-byte interleave), and
! 19: produces a final checksum based on the end state of those 8 streams. If
! 20: parallelization of MD5 hashing is available, the performance gain (or
! 21: CPU usage decrease) is 2x to 6x compared to traditional MD5.
! 22:
! 23: The rsync version on both ends of the connection need MD5P8 support
! 24: built-in for it to be used.
! 25:
! 26: xxHash is still preferred (and faster), but this provides a reasonably
! 27: fast fallback for the case where xxHash libraries are not available at
! 28: build time.
! 29:
! 30: based-on: e94bad1c156fc3910f24e2b3b71a81b0b0bdeb70
! 31: diff --git a/Makefile.in b/Makefile.in
! 32: --- a/Makefile.in
! 33: +++ b/Makefile.in
! 34: @@ -29,14 +29,14 @@ SHELL=/bin/sh
! 35: .SUFFIXES:
! 36: .SUFFIXES: .c .o
! 37:
! 38: -SIMD_x86_64=simd-checksum-x86_64.o
! 39: +SIMD_x86_64=simd-checksum-x86_64.o simd-md5-parallel-x86_64.o
! 40: ASM_x86_64=lib/md5-asm-x86_64.o
! 41:
! 42: GENFILES=configure.sh aclocal.m4 config.h.in rsync.1 rsync.1.html \
! 43: rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html
! 44: HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
! 45: lib/pool_alloc.h lib/mdigest.h lib/md-defines.h version.h
! 46: -LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
! 47: +LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o lib/md5p8.o \
! 48: lib/permstring.o lib/pool_alloc.o lib/sysacls.o lib/sysxattrs.o @LIBOBJS@
! 49: zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
! 50: zlib/trees.o zlib/zutil.o zlib/adler32.o zlib/compress.o zlib/crc32.o
! 51: @@ -140,6 +140,9 @@ git-version.h: mkgitver $(wildcard $(srcdir)/.git/logs/HEAD)
! 52: simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
! 53: @$(srcdir)/cmdormsg disable-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
! 54:
! 55: +simd-md5-parallel-x86_64.o: simd-md5-parallel-x86_64.cpp
! 56: + @$(srcdir)/cmdormsg disable-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-md5-parallel-x86_64.cpp
! 57: +
! 58: lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S config.h lib/md-defines.h
! 59: @$(srcdir)/cmdormsg disable-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
! 60:
! 61: diff --git a/checksum.c b/checksum.c
! 62: --- a/checksum.c
! 63: +++ b/checksum.c
! 64: @@ -52,6 +52,7 @@ struct name_num_obj valid_checksums = {
! 65: { CSUM_XXH64, "xxh64", NULL },
! 66: { CSUM_XXH64, "xxhash", NULL },
! 67: #endif
! 68: + { CSUM_MD5P8, "md5p8", NULL },
! 69: { CSUM_MD5, "md5", NULL },
! 70: { CSUM_MD4, "md4", NULL },
! 71: { CSUM_NONE, "none", NULL },
! 72: @@ -141,6 +142,7 @@ int csum_len_for_type(int cst, BOOL flist_csum)
! 73: case CSUM_MD4_OLD:
! 74: case CSUM_MD4_BUSTED:
! 75: return MD4_DIGEST_LEN;
! 76: + case CSUM_MD5P8:
! 77: case CSUM_MD5:
! 78: return MD5_DIGEST_LEN;
! 79: case CSUM_XXH64:
! 80: @@ -167,6 +169,7 @@ int canonical_checksum(int csum_type)
! 81: case CSUM_MD4_BUSTED:
! 82: break;
! 83: case CSUM_MD4:
! 84: + case CSUM_MD5P8:
! 85: case CSUM_MD5:
! 86: return -1;
! 87: case CSUM_XXH64:
! 88: @@ -179,7 +182,9 @@ int canonical_checksum(int csum_type)
! 89: return 0;
! 90: }
! 91:
! 92: -#ifndef HAVE_SIMD /* See simd-checksum-*.cpp. */
! 93: +#ifdef HAVE_SIMD /* See simd-checksum-*.cpp. */
! 94: +#define get_checksum2 get_checksum2_nosimd
! 95: +#else
! 96: /*
! 97: a simple 32 bit checksum that can be updated from either end
! 98: (inspired by Mark Adler's Adler-32 checksum)
! 99: @@ -200,9 +205,18 @@ uint32 get_checksum1(char *buf1, int32 len)
! 100: }
! 101: return (s1 & 0xffff) + (s2 << 16);
! 102: }
! 103: +
! 104: +void checksum2_enable_prefetch(UNUSED(struct map_struct *map), UNUSED(OFF_T len), UNUSED(int32 blocklen))
! 105: +{
! 106: +}
! 107: +
! 108: +void checksum2_disable_prefetch()
! 109: +{
! 110: +}
! 111: #endif
! 112:
! 113: -void get_checksum2(char *buf, int32 len, char *sum)
! 114: +/* Renamed to get_checksum2_nosimd() with HAVE_SIMD */
! 115: +void get_checksum2(char *buf, int32 len, char *sum, UNUSED(OFF_T prefetch_offset))
! 116: {
! 117: switch (xfersum_type) {
! 118: #ifdef SUPPORT_XXHASH
! 119: @@ -221,6 +235,7 @@ void get_checksum2(char *buf, int32 len, char *sum)
! 120: break;
! 121: }
! 122: #endif
! 123: + case CSUM_MD5P8: /* == CSUM_MD5 for checksum2 */
! 124: case CSUM_MD5: {
! 125: MD5_CTX m5;
! 126: uchar seedbuf[4];
! 127: @@ -373,6 +388,21 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
! 128: break;
! 129: }
! 130: #endif
! 131: + case CSUM_MD5P8: {
! 132: + MD5P8_CTX m5p8;
! 133: +
! 134: + MD5P8_Init(&m5p8);
! 135: +
! 136: + for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
! 137: + MD5P8_Update(&m5p8, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
! 138: +
! 139: + remainder = (int32)(len - i);
! 140: + if (remainder > 0)
! 141: + MD5P8_Update(&m5p8, (uchar *)map_ptr(buf, i, remainder), remainder);
! 142: +
! 143: + MD5P8_Final((uchar *)sum, &m5p8);
! 144: + break;
! 145: + }
! 146: case CSUM_MD5: {
! 147: MD5_CTX m5;
! 148:
! 149: @@ -445,6 +475,7 @@ static union {
! 150: #endif
! 151: MD5_CTX m5;
! 152: } ctx;
! 153: +static MD5P8_CTX m5p8;
! 154: #ifdef SUPPORT_XXHASH
! 155: static XXH64_state_t* xxh64_state;
! 156: #endif
! 157: @@ -481,6 +512,9 @@ void sum_init(int csum_type, int seed)
! 158: XXH3_128bits_reset(xxh3_state);
! 159: break;
! 160: #endif
! 161: + case CSUM_MD5P8:
! 162: + MD5P8_Init(&m5p8);
! 163: + break;
! 164: case CSUM_MD5:
! 165: MD5_Init(&ctx.m5);
! 166: break;
! 167: @@ -531,6 +565,9 @@ void sum_update(const char *p, int32 len)
! 168: XXH3_128bits_update(xxh3_state, p, len);
! 169: break;
! 170: #endif
! 171: + case CSUM_MD5P8:
! 172: + MD5P8_Update(&m5p8, (uchar *)p, len);
! 173: + break;
! 174: case CSUM_MD5:
! 175: MD5_Update(&ctx.m5, (uchar *)p, len);
! 176: break;
! 177: @@ -596,6 +633,9 @@ int sum_end(char *sum)
! 178: break;
! 179: }
! 180: #endif
! 181: + case CSUM_MD5P8:
! 182: + MD5P8_Final((uchar *)sum, &m5p8);
! 183: + break;
! 184: case CSUM_MD5:
! 185: MD5_Final((uchar *)sum, &ctx.m5);
! 186: break;
! 187: diff --git a/generator.c b/generator.c
! 188: --- a/generator.c
! 189: +++ b/generator.c
! 190: @@ -729,10 +729,12 @@ static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
! 191: if (append_mode > 0 && f_copy < 0)
! 192: return 0;
! 193:
! 194: - if (len > 0)
! 195: + if (len > 0) {
! 196: mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
! 197: - else
! 198: + checksum2_enable_prefetch(mapbuf, len, sum.blength);
! 199: + } else {
! 200: mapbuf = NULL;
! 201: + }
! 202:
! 203: for (i = 0; i < sum.count; i++) {
! 204: int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
! 205: @@ -750,7 +752,7 @@ static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
! 206: }
! 207:
! 208: sum1 = get_checksum1(map, n1);
! 209: - get_checksum2(map, n1, sum2);
! 210: + get_checksum2(map, n1, sum2, offset - n1);
! 211:
! 212: if (DEBUG_GTE(DELTASUM, 3)) {
! 213: rprintf(FINFO,
! 214: @@ -762,8 +764,10 @@ static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
! 215: write_buf(f_out, sum2, sum.s2length);
! 216: }
! 217:
! 218: - if (mapbuf)
! 219: + if (mapbuf) {
! 220: unmap_file(mapbuf);
! 221: + checksum2_disable_prefetch();
! 222: + }
! 223:
! 224: return 0;
! 225: }
! 226: diff --git a/lib/md-defines.h b/lib/md-defines.h
! 227: --- a/lib/md-defines.h
! 228: +++ b/lib/md-defines.h
! 229: @@ -15,3 +15,4 @@
! 230: #define CSUM_XXH64 6
! 231: #define CSUM_XXH3_64 7
! 232: #define CSUM_XXH3_128 8
! 233: +#define CSUM_MD5P8 9
! 234: diff --git a/lib/md5p8.c b/lib/md5p8.c
! 235: new file mode 100644
! 236: --- /dev/null
! 237: +++ b/lib/md5p8.c
! 238: @@ -0,0 +1,128 @@
! 239: +/*
! 240: + * MD5-based hash friendly to parallel processing, reference implementation
! 241: + *
! 242: + * Author: Jorrit Jongma, 2020
! 243: + *
! 244: + * Released in the public domain falling back to the MIT license
! 245: + * ( http://www.opensource.org/licenses/MIT ) in case public domain does not
! 246: + * apply in your country.
! 247: + */
! 248: +/*
! 249: + * MD5P8 is an MD5-based hash friendly to parallel processing. The input
! 250: + * stream is divided into 8 independent streams. For each 512 bytes of input,
! 251: + * the first 64 bytes are send to the first stream, the second 64 bytes to
! 252: + * the second stream, etc. The input stream is padded with zeros to the next
! 253: + * multiple of 512 bytes, then a normal MD5 hash is computed on a buffer
! 254: + * containing the A, B, C, and D states of the 8 individual streams, followed
! 255: + * by the (unpadded) length of the input.
! 256: + *
! 257: + * On non-SIMD accelerated CPUs the performance of MD5P8 is slightly lower
! 258: + * than normal MD5 (particularly on files smaller than 10 kB), but with
! 259: + * SIMD-based parallel processing it can be two to six times as fast. Even in
! 260: + * the best-case scenario, xxHash is still at least twice as fast and should
! 261: + * be preferred when available.
! 262: + */
! 263: +
! 264: +#include "rsync.h"
! 265: +
! 266: +#ifdef HAVE_SIMD
! 267: +#define MD5P8_Init MD5P8_Init_c
! 268: +#define MD5P8_Update MD5P8_Update_c
! 269: +#define MD5P8_Final MD5P8_Final_c
! 270: +#endif
! 271: +
! 272: +/* each MD5_CTX needs to be 8-byte aligned */
! 273: +#define MD5P8_Contexts_c(ctx, index) ((MD5_CTX*)((((uintptr_t)((ctx)->context_storage) + 7) & ~7) + (index)*((sizeof(MD5_CTX) + 7) & ~7)))
! 274: +
! 275: +void MD5P8_Init(MD5P8_CTX *ctx)
! 276: +{
! 277: + int i;
! 278: + for (i = 0; i < 8; i++) {
! 279: + MD5_Init(MD5P8_Contexts_c(ctx, i));
! 280: + }
! 281: + ctx->used = 0;
! 282: + ctx->next = 0;
! 283: +}
! 284: +
! 285: +void MD5P8_Update(MD5P8_CTX *ctx, const uchar *input, uint32 length)
! 286: +{
! 287: + uint32 pos = 0;
! 288: +
! 289: + if ((ctx->used) || (length < 64)) {
! 290: + int cpy = MIN(length, 64 - ctx->used);
! 291: + memmove(&ctx->buffer[ctx->used], input, cpy);
! 292: + ctx->used += cpy;
! 293: + length -= cpy;
! 294: + pos += cpy;
! 295: +
! 296: + if (ctx->used == 64) {
! 297: + MD5_Update(MD5P8_Contexts_c(ctx, ctx->next), ctx->buffer, 64);
! 298: + ctx->used = 0;
! 299: + ctx->next = (ctx->next + 1) % 8;
! 300: + }
! 301: + }
! 302: +
! 303: + while (length >= 64) {
! 304: + MD5_Update(MD5P8_Contexts_c(ctx, ctx->next), &input[pos], 64);
! 305: + ctx->next = (ctx->next + 1) % 8;
! 306: + pos += 64;
! 307: + length -= 64;
! 308: + }
! 309: +
! 310: + if (length) {
! 311: + memcpy(ctx->buffer, &input[pos], length);
! 312: + ctx->used = length;
! 313: + }
! 314: +}
! 315: +
! 316: +void MD5P8_Final(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx)
! 317: +{
! 318: + int i;
! 319: + uint32 low = 0, high = 0, sub = ctx->used ? 64 - ctx->used : 0;
! 320: + if (ctx->used) {
! 321: + uchar tmp[64];
! 322: + memset(tmp, 0, 64);
! 323: + MD5P8_Update(ctx, tmp, 64 - ctx->used);
! 324: + }
! 325: + memset(ctx->buffer, 0, 64);
! 326: + while (ctx->next != 0) {
! 327: + MD5P8_Update(ctx, ctx->buffer, 64);
! 328: + sub += 64;
! 329: + }
! 330: +
! 331: + uchar state[34*4] = {0};
! 332: +
! 333: + for (i = 0; i < 8; i++) {
! 334: + MD5_CTX* md = MD5P8_Contexts_c(ctx, i);
! 335: +#ifdef USE_OPENSSL
! 336: + if (low + md->Nl < low) high++;
! 337: + low += md->Nl;
! 338: + high += md->Nh;
! 339: +#else
! 340: + if (low + md->totalN < low) high++;
! 341: + low += md->totalN;
! 342: + high += md->totalN2;
! 343: +#endif
! 344: + SIVALu(state, i*16, md->A);
! 345: + SIVALu(state, i*16 + 4, md->B);
! 346: + SIVALu(state, i*16 + 8, md->C);
! 347: + SIVALu(state, i*16 + 12, md->D);
! 348: + }
! 349: +
! 350: +#ifndef USE_OPENSSL
! 351: + high = (low >> 29) | (high << 3);
! 352: + low = (low << 3);
! 353: +#endif
! 354: +
! 355: + sub <<= 3;
! 356: + if (low - sub > low) high--;
! 357: + low -= sub;
! 358: +
! 359: + SIVALu(state, 32*4, low);
! 360: + SIVALu(state, 33*4, high);
! 361: +
! 362: + MD5_CTX md;
! 363: + MD5_Init(&md);
! 364: + MD5_Update(&md, state, 34*4);
! 365: + MD5_Final(digest, &md);
! 366: +}
! 367: diff --git a/lib/mdigest.h b/lib/mdigest.h
! 368: --- a/lib/mdigest.h
! 369: +++ b/lib/mdigest.h
! 370: @@ -27,3 +27,14 @@ void md5_begin(md_context *ctx);
! 371: void md5_update(md_context *ctx, const uchar *input, uint32 length);
! 372: void md5_result(md_context *ctx, uchar digest[MD5_DIGEST_LEN]);
! 373: #endif
! 374: +
! 375: +typedef struct {
! 376: + uchar context_storage[1024];
! 377: + uchar buffer[512];
! 378: + unsigned int used;
! 379: + unsigned int next;
! 380: +} MD5P8_CTX;
! 381: +
! 382: +void MD5P8_Init(MD5P8_CTX *ctx);
! 383: +void MD5P8_Update(MD5P8_CTX *ctx, const uchar *input, uint32 length);
! 384: +void MD5P8_Final(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx);
! 385: diff --git a/match.c b/match.c
! 386: --- a/match.c
! 387: +++ b/match.c
! 388: @@ -164,6 +164,8 @@ static void hash_search(int f,struct sum_struct *s,
! 389: if (DEBUG_GTE(DELTASUM, 3))
! 390: rprintf(FINFO, "sum=%.8x k=%ld\n", sum, (long)k);
! 391:
! 392: + checksum2_enable_prefetch(buf, len, s->blength);
! 393: +
! 394: offset = aligned_offset = aligned_i = 0;
! 395:
! 396: end = len + 1 - s->sums[s->count-1].len;
! 397: @@ -226,7 +228,7 @@ static void hash_search(int f,struct sum_struct *s,
! 398:
! 399: if (!done_csum2) {
! 400: map = (schar *)map_ptr(buf,offset,l);
! 401: - get_checksum2((char *)map,l,sum2);
! 402: + get_checksum2((char *)map, l, sum2, offset);
! 403: done_csum2 = 1;
! 404: }
! 405:
! 406: @@ -268,7 +270,7 @@ static void hash_search(int f,struct sum_struct *s,
! 407: sum = get_checksum1((char *)map, l);
! 408: if (sum != s->sums[i].sum1)
! 409: goto check_want_i;
! 410: - get_checksum2((char *)map, l, sum2);
! 411: + get_checksum2((char *)map, l, sum2, aligned_offset);
! 412: if (memcmp(sum2, s->sums[i].sum2, s->s2length) != 0)
! 413: goto check_want_i;
! 414: /* OK, we have a re-alignment match. Bump the offset
! 415: @@ -335,6 +337,8 @@ static void hash_search(int f,struct sum_struct *s,
! 416: matched(f, s, buf, offset - s->blength, -2);
! 417: } while (++offset < end);
! 418:
! 419: + checksum2_disable_prefetch();
! 420: +
! 421: matched(f, s, buf, len, -1);
! 422: map_ptr(buf, len-1, 1);
! 423: }
! 424: diff --git a/simd-checksum-x86_64.cpp b/simd-checksum-x86_64.cpp
! 425: --- a/simd-checksum-x86_64.cpp
! 426: +++ b/simd-checksum-x86_64.cpp
! 427: @@ -49,13 +49,33 @@
! 428: * use of the target attribute, selecting the fastest code path based on
! 429: * dispatch priority (GCC 5) or runtime detection of CPU capabilities (GCC 6+).
! 430: * GCC 4.x are not supported to ease configure.ac logic.
! 431: + *
! 432: + * ----
! 433: + *
! 434: + * get_checksum2() is optimized for the case where the selected transfer
! 435: + * checksum is MD5. MD5 can't be made significantly faster with SIMD
! 436: + * instructions than the assembly version already included but SIMD
! 437: + * instructions can be used to hash multiple streams in parallel (see
! 438: + * simd-md5-parallel-x86_64.cpp for details and benchmarks). As rsync's
! 439: + * block-matching algorithm hashes the blocks independently (in contrast to
! 440: + * the whole-file checksum) this method can be employed here.
! 441: + *
! 442: + * To prevent needing to modify the core rsync sources significantly, a
! 443: + * prefetching strategy is used. When a checksum2 is requested, the code
! 444: + * reads ahead several blocks, creates the MD5 hashes for each block in
! 445: + * parallel, returns the hash for the first block, and caches the results
! 446: + * for the other blocks to return in future calls to get_checksum2().
! 447: */
! 448:
! 449: #ifdef __x86_64__
! 450: #ifdef __cplusplus
! 451:
! 452: +extern "C" {
! 453: +
! 454: #include "rsync.h"
! 455:
! 456: +}
! 457: +
! 458: #ifdef HAVE_SIMD
! 459:
! 460: #include <immintrin.h>
! 461: @@ -480,9 +500,235 @@ uint32 get_checksum1(char *buf1, int32 len)
! 462: return get_checksum1_cpp(buf1, len);
! 463: }
! 464:
! 465: -} // extern "C"
! 466: +#if !defined(BENCHMARK_SIMD_CHECKSUM1)
! 467:
! 468: -#ifdef BENCHMARK_SIMD_CHECKSUM1
! 469: +// see simd-md5-parallel-x86_64.cpp
! 470: +extern int md5_parallel_slots();
! 471: +extern int md5_parallel(int streams, char** buf, int* len, char** sum, char* pre4, char* post4);
! 472: +
! 473: +#endif /* !BENCHMARK_SIMD_CHECKSUM1 */
! 474: +
! 475: +#if !defined(BENCHMARK_SIMD_CHECKSUM1) && !defined(BENCHMARK_SIMD_CHECKSUM2)
! 476: +
! 477: +#define PREFETCH_ENABLE 1 // debugging
! 478: +
! 479: +#if 0 // debugging
! 480: +#define PREFETCH_PRINTF(f_, ...) printf((f_), ##__VA_ARGS__)
! 481: +#else
! 482: +#define PREFETCH_PRINTF(f_, ...) (void)0;
! 483: +#endif
! 484: +
! 485: +#define PREFETCH_MIN_LEN 1024 // the overhead is unlikely to be worth the gain for small blocks
! 486: +#define PREFETCH_MAX_BLOCKS 8
! 487: +
! 488: +typedef struct {
! 489: + int in_use;
! 490: + OFF_T offset;
! 491: + int32 len;
! 492: + char sum[SUM_LENGTH];
! 493: +} prefetch_sum_t;
! 494: +
! 495: +typedef struct {
! 496: + struct map_struct *map;
! 497: + OFF_T len;
! 498: + OFF_T last;
! 499: + int32 blocklen;
! 500: + int blocks;
! 501: + prefetch_sum_t sums[PREFETCH_MAX_BLOCKS];
! 502: +} prefetch_t;
! 503: +
! 504: +prefetch_t *prefetch;
! 505: +
! 506: +extern int xfersum_type;
! 507: +extern int checksum_seed;
! 508: +extern int proper_seed_order;
! 509: +extern void get_checksum2_nosimd(char *buf, int32 len, char *sum, OFF_T prefetch_offset);
! 510: +
! 511: +extern char *map_ptr(struct map_struct *map, OFF_T offset, int32 len);
! 512: +
! 513: +void checksum2_disable_prefetch()
! 514: +{
! 515: + if (prefetch) {
! 516: + PREFETCH_PRINTF("checksum2_disable_prefetch\n");
! 517: + free(prefetch);
! 518: + prefetch = NULL;
! 519: + }
! 520: +}
! 521: +
! 522: +void checksum2_enable_prefetch(UNUSED(struct map_struct *map), UNUSED(OFF_T len), UNUSED(int32 blocklen))
! 523: +{
! 524: +#ifdef PREFETCH_ENABLE
! 525: + checksum2_disable_prefetch();
! 526: + int slots = md5_parallel_slots();
! 527: + if ((xfersum_type == CSUM_MD5 || xfersum_type == CSUM_MD5P8) && slots > 1 && len >= blocklen * PREFETCH_MAX_BLOCKS && blocklen >= PREFETCH_MIN_LEN) {
! 528: + prefetch = (prefetch_t*)malloc(sizeof(prefetch_t));
! 529: + memset(prefetch, 0, sizeof(prefetch_t));
! 530: + prefetch->map = map;
! 531: + prefetch->len = len;
! 532: + prefetch->last = 0;
! 533: + prefetch->blocklen = blocklen;
! 534: + prefetch->blocks = MIN(PREFETCH_MAX_BLOCKS, slots);
! 535: + PREFETCH_PRINTF("checksum2_enable_prefetch len:%ld blocklen:%d blocks:%d\n", prefetch->len, prefetch->blocklen, prefetch->blocks);
! 536: + }
! 537: +#endif
! 538: +}
! 539: +
! 540: +static inline void checksum2_reset_prefetch()
! 541: +{
! 542: + for (int i = 0; i < PREFETCH_MAX_BLOCKS; i++) {
! 543: + prefetch->sums[i].in_use = 0;
! 544: + }
! 545: +}
! 546: +
! 547: +static int get_checksum2_prefetched(int32 len, char* sum, OFF_T prefetch_offset)
! 548: +{
! 549: + if (prefetch->sums[0].in_use) {
! 550: + if ((prefetch->sums[0].offset == prefetch_offset) && (prefetch->sums[0].len == len)) {
! 551: + memcpy(sum, prefetch->sums[0].sum, SUM_LENGTH);
! 552: + for (int i = 0; i < PREFETCH_MAX_BLOCKS - 1; i++) {
! 553: + prefetch->sums[i] = prefetch->sums[i + 1];
! 554: + }
! 555: + prefetch->sums[PREFETCH_MAX_BLOCKS - 1].in_use = 0;
! 556: + PREFETCH_PRINTF("checksum2_prefetch HIT len:%d offset:%ld\n", len, prefetch_offset);
! 557: + return 1;
! 558: + } else {
! 559: + // unexpected access, reset cache
! 560: + PREFETCH_PRINTF("checksum2_prefetch MISS len:%d offset:%ld\n", len, prefetch_offset);
! 561: + checksum2_reset_prefetch();
! 562: + }
! 563: + }
! 564: + return 0;
! 565: +}
! 566: +
! 567: +static int checksum2_perform_prefetch(OFF_T prefetch_offset)
! 568: +{
! 569: + int blocks = MIN(MAX(1, (prefetch->len + prefetch->blocklen - 1) / prefetch->blocklen), prefetch->blocks);
! 570: + if (blocks < 2) return 0; // fall through to non-simd, probably faster
! 571: +
! 572: + int32 total = 0;
! 573: + int i;
! 574: + for (i = 0; i < blocks; i++) {
! 575: + prefetch->sums[i].offset = prefetch_offset + total;
! 576: + prefetch->sums[i].len = MIN(prefetch->blocklen, prefetch->len - prefetch_offset - total);
! 577: + prefetch->sums[i].in_use = 0;
! 578: + total += prefetch->sums[i].len;
! 579: + }
! 580: + for (; i < PREFETCH_MAX_BLOCKS; i++) {
! 581: + prefetch->sums[i].in_use = 0;
! 582: + }
! 583: +
! 584: + uchar seedbuf[4];
! 585: + SIVALu(seedbuf, 0, checksum_seed);
! 586: +
! 587: + PREFETCH_PRINTF("checksum2_perform_prefetch pos:%ld len:%d blocks:%d\n", prefetch_offset, total, blocks);
! 588: + char* mapbuf = map_ptr(prefetch->map, prefetch_offset, total);
! 589: + char* bufs[PREFETCH_MAX_BLOCKS] = {0};
! 590: + int lens[PREFETCH_MAX_BLOCKS] = {0};
! 591: + char* sums[PREFETCH_MAX_BLOCKS] = {0};
! 592: + for (i = 0; i < blocks; i++) {
! 593: + bufs[i] = mapbuf + prefetch->sums[i].offset - prefetch_offset;
! 594: + lens[i] = prefetch->sums[i].len;
! 595: + sums[i] = prefetch->sums[i].sum;
! 596: + }
! 597: + if (md5_parallel(blocks, bufs, lens, sums, (proper_seed_order && checksum_seed) ? (char*)seedbuf : NULL, (!proper_seed_order && checksum_seed) ? (char*)seedbuf : NULL)) {
! 598: + for (i = 0; i < blocks; i++) {
! 599: + prefetch->sums[i].in_use = 1;
! 600: + }
! 601: + return 1;
! 602: + } else {
! 603: + // this should never be, abort
! 604: + PREFETCH_PRINTF("checksum2_perform_prefetch PMD5 ABORT\n");
! 605: + checksum2_disable_prefetch();
! 606: + }
! 607: + return 0;
! 608: +}
! 609: +
! 610: +void get_checksum2(char *buf, int32 len, char *sum, OFF_T prefetch_offset)
! 611: +{
! 612: + if (prefetch) {
! 613: + PREFETCH_PRINTF("get_checksum2 %d @ %ld\n", len, prefetch_offset);
! 614: + OFF_T last = prefetch->last;
! 615: + prefetch->last = prefetch_offset;
! 616: + if ((prefetch_offset != 0) && (prefetch_offset != last + prefetch->blocklen)) {
! 617: + // we're looking around trying to align blocks, prefetching will slow things down
! 618: + PREFETCH_PRINTF("get_checksum2 SEEK\n");
! 619: + checksum2_reset_prefetch();
! 620: + } else if (get_checksum2_prefetched(len, sum, prefetch_offset)) {
! 621: + // hit
! 622: + return;
! 623: + } else if (checksum2_perform_prefetch(prefetch_offset)) {
! 624: + if (get_checksum2_prefetched(len, sum, prefetch_offset)) {
! 625: + // hit; should always be as we just fetched this data
! 626: + return;
! 627: + } else {
! 628: + // this should never be, abort
! 629: + PREFETCH_PRINTF("get_checksum2 MISSING DATA ABORT\n");
! 630: + checksum2_disable_prefetch();
! 631: + }
! 632: + }
! 633: + }
! 634: + get_checksum2_nosimd(buf, len, sum, prefetch_offset);
! 635: +}
! 636: +#endif /* !BENCHMARK_SIMD_CHECKSUM1 && !BENCHMARK_SIMD_CHECKSUM2 */
! 637: +
! 638: +} // "C"
! 639: +
! 640: +/* Benchmark compilation
! 641: +
! 642: + The get_checksum1() benchmark runs through all available code paths in a
! 643: + single execution, the get_checksum2()/MD5 and MD5P8 benchmark needs to be
! 644: + recompiled for each code path (it always uses the fastest path available
! 645: + on the current CPU otherwise). Note that SSE2/AVX2 MD5 optimizations will
! 646: + be used when applicable regardless of rsync being built with OpenSSL.
! 647: +
! 648: + Something like the following should compile and run the benchmarks:
! 649: +
! 650: + # if gcc
! 651: + export CC=gcc
! 652: + export CXX=g++
! 653: + export CXX_BASE="-g -O3 -fno-exceptions -fno-rtti"
! 654: +
! 655: + # else if clang
! 656: + export CC=clang
! 657: + export CXX=clang++
! 658: + export CXX_BASE="-g -O3 -fno-exceptions -fno-rtti -fno-slp-vectorize"
! 659: +
! 660: + # /if
! 661: +
! 662: + export CONF_EXTRA="--disable-md2man --disable-zstd --disable-lz4 --disable-xxhash"
! 663: + export CXX_CSUM1="$CXX_BASE simd-checksum-x86_64.cpp"
! 664: + export CXX_MD5P="$CXX_BASE -c -o simd-md5-parallel-x86_64.o simd-md5-parallel-x86_64.cpp"
! 665: + export CXX_CSUM2="$CXX_BASE simd-checksum-x86_64.cpp simd-md5-parallel-x86_64.o lib/md5.o lib/md5p8.o lib/md5-asm-x86_64.o"
! 666: +
! 667: + rm bench_csum*
! 668: +
! 669: + ./configure --disable-openssl --enable-simd $CONF_EXTRA && make clean && make -j4
! 670: +
! 671: + $CXX -DBENCHMARK_SIMD_CHECKSUM1 $CXX_CSUM1 -o bench_csum1.all
! 672: +
! 673: + $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_MD5P
! 674: + $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_CSUM2 -o bench_csum2.asm
! 675: +
! 676: + $CXX -DBENCHMARK_SIMD_CHECKSUM2 -DPMD5_ALLOW_SSE2 $CXX_MD5P
! 677: + $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_CSUM2 -o bench_csum2.sse2
! 678: +
! 679: + $CXX -DBENCHMARK_SIMD_CHECKSUM2 -DPMD5_ALLOW_AVX2 $CXX_MD5P
! 680: + $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_CSUM2 -o bench_csum2.avx2
! 681: +
! 682: + ./configure --enable-openssl --enable-simd $CONF_EXTRA && make clean && make -j4
! 683: +
! 684: + $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_MD5P
! 685: + $CXX -DBENCHMARK_SIMD_CHECKSUM2 $CXX_CSUM2 -o bench_csum2.openssl -lcrypto
! 686: +
! 687: + ./bench_csum1.all
! 688: + ./bench_csum2.asm
! 689: + ./bench_csum2.openssl
! 690: + ./bench_csum2.sse2
! 691: + ./bench_csum2.avx2
! 692: +
! 693: + */
! 694: +
! 695: +#if defined(BENCHMARK_SIMD_CHECKSUM1) || defined(BENCHMARK_SIMD_CHECKSUM2)
! 696: #pragma clang optimize off
! 697: #pragma GCC push_options
! 698: #pragma GCC optimize ("O0")
! 699: @@ -493,7 +739,9 @@ uint32 get_checksum1(char *buf1, int32 len)
! 700: #ifndef CLOCK_MONOTONIC_RAW
! 701: #define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC
! 702: #endif
! 703: +#endif /* BENCHMARK_SIMD_CHECKSUM1 || BENCHMARK_SIMD_CHECKSUM2 */
! 704:
! 705: +#ifdef BENCHMARK_SIMD_CHECKSUM1
! 706: static void benchmark(const char* desc, int32 (*func)(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2), schar* buf, int32 len) {
! 707: struct timespec start, end;
! 708: uint64_t us;
! 709: @@ -509,7 +757,7 @@ static void benchmark(const char* desc, int32 (*func)(schar* buf, int32 len, int
! 710: clock_gettime(CLOCK_MONOTONIC_RAW, &end);
! 711: us = next == 0 ? 0 : (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_nsec - start.tv_nsec) / 1000;
! 712: cs = next == 0 ? 0 : (s1 & 0xffff) + (s2 << 16);
! 713: - printf("%-5s :: %5.0f MB/s :: %08x\n", desc, us ? (float)(len / (1024 * 1024) * ROUNDS) / ((float)us / 1000000.0f) : 0, cs);
! 714: + printf("CSUM1 :: %-5s :: %5.0f MB/s :: %08x\n", desc, us ? (float)(len / (1024 * 1024) * ROUNDS) / ((float)us / 1000000.0f) : 0, cs);
! 715: }
! 716:
! 717: static int32 get_checksum1_auto(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) {
! 718: @@ -533,10 +781,108 @@ int main() {
! 719: free(buf);
! 720: return 0;
! 721: }
! 722: +#endif /* BENCHMARK_SIMD_CHECKSUM1 */
! 723:
! 724: +#ifdef BENCHMARK_SIMD_CHECKSUM2
! 725: +static void benchmark(const char* desc, void (*func)(char* buf, int32 len, char* sum_out), void (*func2)(char* buf, int32 len, char* sum_out), char* buf, int32 len, int streams) {
! 726: + struct timespec start, end;
! 727: + uint64_t us;
! 728: + unsigned char cs1[16];
! 729: + unsigned char cs2[16];
! 730: + int i;
! 731: +
! 732: + clock_gettime(CLOCK_MONOTONIC_RAW, &start);
! 733: + for (i = 0; i < ROUNDS; i++) {
! 734: + func(buf, len, (char*)cs1);
! 735: + }
! 736: + clock_gettime(CLOCK_MONOTONIC_RAW, &end);
! 737: + us = (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_nsec - start.tv_nsec) / 1000;
! 738: +
! 739: + func2(buf, len, (char*)cs2);
! 740: +
! 741: + float perf = us ? (float)(len / (1024 * 1024) * ROUNDS) / ((float)us / 1000000.0f) : 0;
! 742: + printf("CSUM2 :: %-7s :: %5.0f to %5.0f MB/s :: ", desc, perf, perf * streams);
! 743: + for (i = 0; i < 16; i++) {
! 744: + printf("%02x", cs1[i] & 0xFF);
! 745: + }
! 746: + printf(" :: ");
! 747: + for (i = 0; i < 16; i++) {
! 748: + printf("%02x", cs2[i] & 0xFF);
! 749: + }
! 750: + printf("\n");
! 751: +}
! 752: +
! 753: +static void benchmark_inner(char* buf, int32 len, char* sum_out) {
! 754: + // This should produce the same output for different optimizations
! 755: + // levels, not the same as sanity_check()
! 756: +
! 757: + char* bufs[8] = {0};
! 758: + int lens[8] = {0};
! 759: + char* sums[8] = {0};
! 760: +
! 761: + bufs[0] = buf;
! 762: + lens[0] = len;
! 763: + sums[0] = sum_out;
! 764: + md5_parallel(1, bufs, lens, sums, NULL, NULL);
! 765: +}
! 766: +
! 767: +extern "C" {
! 768: +extern void MD5P8_Init_c(MD5P8_CTX *ctx);
! 769: +extern void MD5P8_Update_c(MD5P8_CTX *ctx, const uchar *input, uint32 length);
! 770: +extern void MD5P8_Final_c(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx);
! 771: +}
! 772: +
! 773: +static void sanity_check(char* buf, int32 len, char* sum_out) {
! 774: + // This should produce the same output for different optimizations
! 775: + // levels, not the same as benchmark_inner()
! 776: + if (md5_parallel_slots() <= 1) {
! 777: + MD5P8_CTX m5p8;
! 778: + MD5P8_Init_c(&m5p8);
! 779: + MD5P8_Update_c(&m5p8, (uchar *)buf, len);
! 780: + MD5P8_Final_c((uchar *)sum_out, &m5p8);
! 781: + } else {
! 782: + MD5P8_CTX m5p8;
! 783: + MD5P8_Init(&m5p8);
! 784: + MD5P8_Update(&m5p8, (uchar *)buf, len);
! 785: + MD5P8_Final((uchar *)sum_out, &m5p8);
! 786: + }
! 787: +}
! 788: +
! 789: +int main() {
! 790: + // This benchmarks the parallel MD5 checksum rather than get_checksum2()
! 791: + // as the latter would require compiling in a lot of rsync's code, but
! 792: + // it touches all the same internals so the performance should be nearly
! 793: + // identical.
! 794: +
! 795: + int i;
! 796: + char* buf = (char*)malloc(BLOCK_LEN);
! 797: + for (i = 0; i < BLOCK_LEN; i++) buf[i] = (i + (i % 3) + (i % 11)) % 256;
! 798: +
! 799: + const char* method = "?";
! 800: + switch (md5_parallel_slots()) {
! 801: + case 8: method = "AVX2"; break;
! 802: + case 4: method = "SSE2"; break;
! 803: +#ifdef USE_OPENSSL
! 804: + case 1: method = "OpenSSL"; break;
! 805: +#elif (CSUM_CHUNK == 64)
! 806: + case 1: method = "ASM"; break;
! 807: +#else
! 808: + // this won't happen unless you modified code somewhere
! 809: + case 1: method = "Raw-C"; break;
! 810: +#endif
! 811: + }
! 812: +
! 813: + benchmark(method, benchmark_inner, sanity_check, buf, BLOCK_LEN, md5_parallel_slots());
! 814: +
! 815: + free(buf);
! 816: + return 0;
! 817: +}
! 818: +#endif /* BENCHMARK_SIMD_CHECKSUM2 */
! 819: +
! 820: +#if defined(BENCHMARK_SIMD_CHECKSUM1) || defined(BENCHMARK_SIMD_CHECKSUM2)
! 821: #pragma GCC pop_options
! 822: #pragma clang optimize on
! 823: -#endif /* BENCHMARK_SIMD_CHECKSUM1 */
! 824: +#endif /* BENCHMARK_SIMD_CHECKSUM1 || BENCHMARK_SIMD_CHECKSUM2 */
! 825:
! 826: #endif /* HAVE_SIMD */
! 827: #endif /* __cplusplus */
! 828: diff --git a/simd-md5-parallel-x86_64.cpp b/simd-md5-parallel-x86_64.cpp
! 829: new file mode 100644
! 830: --- /dev/null
! 831: +++ b/simd-md5-parallel-x86_64.cpp
! 832: @@ -0,0 +1,1138 @@
! 833: +/*
! 834: + * SSE2/AVX2-optimized routines to process multiple MD5 streams in parallel.
! 835: + *
! 836: + * Original author: Nicolas Noble, 2017
! 837: + * Modifications: Jorrit Jongma, 2020
! 838: + *
! 839: + * The original code was released in the public domain by the original author,
! 840: + * falling back to the MIT license ( http://www.opensource.org/licenses/MIT )
! 841: + * in case public domain does not apply in your country. These modifications
! 842: + * are likewise released in the public domain, with the same MIT license
! 843: + * fallback.
! 844: + *
! 845: + * The original publication can be found at:
! 846: + *
! 847: + * https://github.com/nicolasnoble/sse-hash
! 848: + */
! 849: +/*
! 850: + * Nicolas' original code has been extended to add AVX2 support, all non-SIMD
! 851: + * MD5 code has been removed and those code paths rerouted to use the MD5
! 852: + * code already present in rsync, and wrapper functions have been added. The
! 853: + * MD5P8 code is also new, and is the reason for the new stride parameter.
! 854: + *
! 855: + * This code allows multiple independent MD5 streams to be processed in
! 856: + * parallel, 4 with SSE2, 8 with AVX2. While single-stream performance is
! 857: + * lower than that of the original C routines for MD5, the processing of
! 858: + * additional streams is "for free".
! 859: + *
! 860: + * Single streams are rerouted to rsync's normal MD5 code as that is faster
! 861: + * for that case. A further optimization is possible by using SSE2 code on
! 862: + * AVX2-supporting CPUs when the number of streams is 2, 3, or 4. This is not
! 863: + * implemented here as it would require some restructuring, and in practise
! 864: + * the code here is only rarely called with less than the maximum amount of
! 865: + * streams (typically once at the end of each checksum2'd file).
! 866: + *
! 867: + * Benchmarks (in MB/s) C ASM SSE2*1 SSE2*4 AVX2*1 AVX2*8
! 868: + * - Intel Atom D2700 302 334 166 664 N/A N/A
! 869: + * - Intel i7-7700hq 351 376 289 1156 273 2184
! 870: + * - AMD ThreadRipper 2950x 728 784 568 2272 430 3440
! 871: + */
! 872: +
! 873: +#ifdef __x86_64__
! 874: +#ifdef __cplusplus
! 875: +
! 876: +extern "C" {
! 877: +
! 878: +#include "rsync.h"
! 879: +
! 880: +}
! 881: +
! 882: +#ifdef HAVE_SIMD
! 883: +
! 884: +#ifndef BENCHMARK_SIMD_CHECKSUM2
! 885: +#define PMD5_ALLOW_SSE2 // debugging
! 886: +#define PMD5_ALLOW_AVX2 // debugging
! 887: +#endif
! 888: +
! 889: +#ifdef PMD5_ALLOW_AVX2
! 890: +#ifndef PMD5_ALLOW_SSE2
! 891: +#define PMD5_ALLOW_SSE2
! 892: +#endif
! 893: +#endif
! 894: +
! 895: +#include <stdint.h>
! 896: +#include <string.h>
! 897: +
! 898: +#include <immintrin.h>
! 899: +
! 900: +/* Some clang versions don't like it when you use static with multi-versioned functions: linker errors */
! 901: +#ifdef __clang__
! 902: +#define MVSTATIC
! 903: +#else
! 904: +#define MVSTATIC static
! 905: +#endif
! 906: +
! 907: +// Missing from the headers on gcc 6 and older, clang 8 and older
! 908: +typedef long long __m128i_u __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));
! 909: +typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, __aligned__(1)));
! 910: +
! 911: +#define PMD5_SLOTS_DEFAULT 0
! 912: +#define PMD5_SLOTS_SSE2 4
! 913: +#define PMD5_SLOTS_AVX2 8
! 914: +#define PMD5_SLOTS_MAX PMD5_SLOTS_AVX2
! 915: +
! 916: +#ifdef PMD5_ALLOW_SSE2
! 917: +__attribute__ ((target("sse2"))) MVSTATIC int pmd5_slots()
! 918: +{
! 919: + return PMD5_SLOTS_SSE2;
! 920: +}
! 921: +#endif
! 922: +
! 923: +#ifdef PMD5_ALLOW_AVX2
! 924: +__attribute__ ((target("avx2"))) MVSTATIC int pmd5_slots()
! 925: +{
! 926: + return PMD5_SLOTS_AVX2;
! 927: +}
! 928: +#endif
! 929: +
! 930: +__attribute__ ((target("default"))) MVSTATIC int pmd5_slots()
! 931: +{
! 932: + return PMD5_SLOTS_DEFAULT;
! 933: +}
! 934: +
! 935: +/* The parallel MD5 context structure. */
! 936: +typedef struct {
! 937: + __m128i state_sse2[4];
! 938: + __m256i state_avx2[4];
! 939: + uint64_t len[PMD5_SLOTS_MAX];
! 940: +} pmd5_context;
! 941: +
! 942: +/* The status returned by the various functions below. */
! 943: +typedef enum {
! 944: + PMD5_SUCCESS,
! 945: + PMD5_INVALID_SLOT,
! 946: + PMD5_UNALIGNED_UPDATE,
! 947: +} pmd5_status;
! 948: +
! 949: +/* Initializes all slots in the given pmd5 context. */
! 950: +__attribute__ ((target("default"))) MVSTATIC pmd5_status pmd5_init_all(pmd5_context * ctx);
! 951: +
! 952: +/* Initializes a single slot out in the given pmd5 context. */
! 953: +static pmd5_status pmd5_init_slot(pmd5_context * ctx, int slot);
! 954: +
! 955: +/* Makes an MD5 update on all slots in parallel, given the same exact length on all streams.
! 956: + The stream pointers will be incremented accordingly.
! 957: + It is valid for a stream pointer to be NULL. Garbage will then be hashed into its corresponding slot.
! 958: + The argument length NEEDS to be a multiple of 64. If not, an error is returned, and the context is corrupted.
! 959: + Stride defaults to 64 if 0 is passed. */
! 960: +static pmd5_status pmd5_update_all_simple(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX], uint64_t length, uint64_t stride);
! 961: +
! 962: +/* Makes an MD5 update on all slots in parallel, given different lengths.
! 963: + The stream pointers will be incremented accordingly.
! 964: + The lengths will be decreased accordingly. Not all data might be consumed.
! 965: + It is valid for a stream pointer to be NULL. Garbage will then be hashed into its corresponding slot.
! 966: + The argument lengths NEEDS to contain only multiples of 64. If not, an error is returned, and the context is corrupted. */
! 967: +static pmd5_status pmd5_update_all(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX], uint64_t lengths[PMD5_SLOTS_MAX]);
! 968: +
! 969: +/* Finishes all slots at once. Fills in all digests. */
! 970: +static pmd5_status pmd5_finish_all(pmd5_context * ctx, uint8_t digests[PMD5_SLOTS_MAX][MD5_DIGEST_LEN]);
! 971: +
! 972: +/* Finishes one slot. The other slots will be unnaffected. The finished slot can then continue to hash garbage using
! 973: + a NULL pointer as its stream argument, or needs to be reinitialized using pmd5_init_slot before being usable again. */
! 974: +static pmd5_status pmd5_finish_slot(pmd5_context * ctx, uint8_t digest[MD5_DIGEST_LEN], int slot);
! 975: +
! 976: +/* Finishes one slot. Extra data is allowed to be passed on as an argument. Length DOESN'T need to be a
! 977: + multiple of 64. The other slots will be unaffected. The finished slot can then continue to hash garbage using
! 978: + a NULL pointer as its stream argument, or needs to be reinitialized using pmd5_init_slot before being usable again. */
! 979: +static pmd5_status pmd5_finish_slot_with_extra(pmd5_context * ctx, uint8_t digest[MD5_DIGEST_LEN], int slot, const uint8_t * data, uint64_t length);
! 980: +
! 981: +/* Insert a normal MD5 context into a given slot of a given parallel MD5 context. */
! 982: +static pmd5_status md5_to_pmd5(const MD5_CTX * ctx, pmd5_context * pctx, int slot);
! 983: +
! 984: +/* Extract a normal MD5 context from a given slot of a given parallel MD5 context. */
! 985: +static pmd5_status pmd5_to_md5(const pmd5_context * pctx, MD5_CTX * ctx, int slot);
! 986: +
! 987: +#define S11 7
! 988: +#define S12 12
! 989: +#define S13 17
! 990: +#define S14 22
! 991: +#define S21 5
! 992: +#define S22 9
! 993: +#define S23 14
! 994: +#define S24 20
! 995: +#define S31 4
! 996: +#define S32 11
! 997: +#define S33 16
! 998: +#define S34 23
! 999: +#define S41 6
! 1000: +#define S42 10
! 1001: +#define S43 15
! 1002: +#define S44 21
! 1003: +
! 1004: +#define T1 0xD76AA478
! 1005: +#define T2 0xE8C7B756
! 1006: +#define T3 0x242070DB
! 1007: +#define T4 0xC1BDCEEE
! 1008: +#define T5 0xF57C0FAF
! 1009: +#define T6 0x4787C62A
! 1010: +#define T7 0xA8304613
! 1011: +#define T8 0xFD469501
! 1012: +#define T9 0x698098D8
! 1013: +#define T10 0x8B44F7AF
! 1014: +#define T11 0xFFFF5BB1
! 1015: +#define T12 0x895CD7BE
! 1016: +#define T13 0x6B901122
! 1017: +#define T14 0xFD987193
! 1018: +#define T15 0xA679438E
! 1019: +#define T16 0x49B40821
! 1020: +#define T17 0xF61E2562
! 1021: +#define T18 0xC040B340
! 1022: +#define T19 0x265E5A51
! 1023: +#define T20 0xE9B6C7AA
! 1024: +#define T21 0xD62F105D
! 1025: +#define T22 0x02441453
! 1026: +#define T23 0xD8A1E681
! 1027: +#define T24 0xE7D3FBC8
! 1028: +#define T25 0x21E1CDE6
! 1029: +#define T26 0xC33707D6
! 1030: +#define T27 0xF4D50D87
! 1031: +#define T28 0x455A14ED
! 1032: +#define T29 0xA9E3E905
! 1033: +#define T30 0xFCEFA3F8
! 1034: +#define T31 0x676F02D9
! 1035: +#define T32 0x8D2A4C8A
! 1036: +#define T33 0xFFFA3942
! 1037: +#define T34 0x8771F681
! 1038: +#define T35 0x6D9D6122
! 1039: +#define T36 0xFDE5380C
! 1040: +#define T37 0xA4BEEA44
! 1041: +#define T38 0x4BDECFA9
! 1042: +#define T39 0xF6BB4B60
! 1043: +#define T40 0xBEBFBC70
! 1044: +#define T41 0x289B7EC6
! 1045: +#define T42 0xEAA127FA
! 1046: +#define T43 0xD4EF3085
! 1047: +#define T44 0x04881D05
! 1048: +#define T45 0xD9D4D039
! 1049: +#define T46 0xE6DB99E5
! 1050: +#define T47 0x1FA27CF8
! 1051: +#define T48 0xC4AC5665
! 1052: +#define T49 0xF4292244
! 1053: +#define T50 0x432AFF97
! 1054: +#define T51 0xAB9423A7
! 1055: +#define T52 0xFC93A039
! 1056: +#define T53 0x655B59C3
! 1057: +#define T54 0x8F0CCC92
! 1058: +#define T55 0xFFEFF47D
! 1059: +#define T56 0x85845DD1
! 1060: +#define T57 0x6FA87E4F
! 1061: +#define T58 0xFE2CE6E0
! 1062: +#define T59 0xA3014314
! 1063: +#define T60 0x4E0811A1
! 1064: +#define T61 0xF7537E82
! 1065: +#define T62 0xBD3AF235
! 1066: +#define T63 0x2AD7D2BB
! 1067: +#define T64 0xEB86D391
! 1068: +
! 1069: +#define ROTL_SSE2(x, n) { \
! 1070: + __m128i s; \
! 1071: + s = _mm_srli_epi32(x, 32 - n); \
! 1072: + x = _mm_slli_epi32(x, n); \
! 1073: + x = _mm_or_si128(x, s); \
! 1074: +};
! 1075: +
! 1076: +#define ROTL_AVX2(x, n) { \
! 1077: + __m256i s; \
! 1078: + s = _mm256_srli_epi32(x, 32 - n); \
! 1079: + x = _mm256_slli_epi32(x, n); \
! 1080: + x = _mm256_or_si256(x, s); \
! 1081: +};
! 1082: +
! 1083: +#define F_SSE2(x, y, z) _mm_or_si128(_mm_and_si128(x, y), _mm_andnot_si128(x, z))
! 1084: +#define G_SSE2(x, y, z) _mm_or_si128(_mm_and_si128(x, z), _mm_andnot_si128(z, y))
! 1085: +#define H_SSE2(x, y, z) _mm_xor_si128(_mm_xor_si128(x, y), z)
! 1086: +#define I_SSE2(x, y, z) _mm_xor_si128(y, _mm_or_si128(x, _mm_andnot_si128(z, _mm_set1_epi32(0xffffffff))))
! 1087: +
! 1088: +#define F_AVX2(x, y, z) _mm256_or_si256(_mm256_and_si256(x, y), _mm256_andnot_si256(x, z))
! 1089: +#define G_AVX2(x, y, z) _mm256_or_si256(_mm256_and_si256(x, z), _mm256_andnot_si256(z, y))
! 1090: +#define H_AVX2(x, y, z) _mm256_xor_si256(_mm256_xor_si256(x, y), z)
! 1091: +#define I_AVX2(x, y, z) _mm256_xor_si256(y, _mm256_or_si256(x, _mm256_andnot_si256(z, _mm256_set1_epi32(0xffffffff))))
! 1092: +
! 1093: +#define SET_SSE2(step, a, b, c, d, x, s, ac) { \
! 1094: + a = _mm_add_epi32(_mm_add_epi32(a, _mm_add_epi32(x, _mm_set1_epi32(T##ac))), step##_SSE2(b, c, d)); \
! 1095: + ROTL_SSE2(a, s); \
! 1096: + a = _mm_add_epi32(a, b); \
! 1097: +}
! 1098: +
! 1099: +#define SET_AVX2(step, a, b, c, d, x, s, ac) { \
! 1100: + a = _mm256_add_epi32(_mm256_add_epi32(a, _mm256_add_epi32(x, _mm256_set1_epi32(T##ac))), step##_AVX2(b, c, d)); \
! 1101: + ROTL_AVX2(a, s); \
! 1102: + a = _mm256_add_epi32(a, b); \
! 1103: +}
! 1104: +
! 1105: +#define IA 0x67452301
! 1106: +#define IB 0xefcdab89
! 1107: +#define IC 0x98badcfe
! 1108: +#define ID 0x10325476
! 1109: +
! 1110: +#define GET_MD5_DATA(dest, src, pos) \
! 1111: + dest = \
! 1112: + ((uint32_t) src[pos + 0]) << 0 | \
! 1113: + ((uint32_t) src[pos + 1]) << 8 | \
! 1114: + ((uint32_t) src[pos + 2]) << 16 | \
! 1115: + ((uint32_t) src[pos + 3]) << 24
! 1116: +
! 1117: +#define GET_PMD5_DATA_SSE2(dest, src, pos) { \
! 1118: + uint32_t v0, v1, v2, v3; \
! 1119: + GET_MD5_DATA(v0, src[0], pos); \
! 1120: + GET_MD5_DATA(v1, src[1], pos); \
! 1121: + GET_MD5_DATA(v2, src[2], pos); \
! 1122: + GET_MD5_DATA(v3, src[3], pos); \
! 1123: + dest = _mm_setr_epi32(v0, v1, v2, v3); \
! 1124: +}
! 1125: +
! 1126: +#define GET_PMD5_DATA_AVX2(dest, src, pos) { \
! 1127: + uint32_t v0, v1, v2, v3; \
! 1128: + uint32_t v4, v5, v6, v7; \
! 1129: + GET_MD5_DATA(v0, src[0], pos); \
! 1130: + GET_MD5_DATA(v1, src[1], pos); \
! 1131: + GET_MD5_DATA(v2, src[2], pos); \
! 1132: + GET_MD5_DATA(v3, src[3], pos); \
! 1133: + GET_MD5_DATA(v4, src[4], pos); \
! 1134: + GET_MD5_DATA(v5, src[5], pos); \
! 1135: + GET_MD5_DATA(v6, src[6], pos); \
! 1136: + GET_MD5_DATA(v7, src[7], pos); \
! 1137: + dest = _mm256_setr_epi32(v0, v1, v2, v3, \
! 1138: + v4, v5, v6, v7); \
! 1139: +}
! 1140: +
! 1141: +#define PUT_MD5_DATA(dest, val, pos) { \
! 1142: + dest[pos + 0] = (val >> 0) & 0xff; \
! 1143: + dest[pos + 1] = (val >> 8) & 0xff; \
! 1144: + dest[pos + 2] = (val >> 16) & 0xff; \
! 1145: + dest[pos + 3] = (val >> 24) & 0xff; \
! 1146: +}
! 1147: +
! 1148: +const static uint8_t md5_padding[64] = {
! 1149: + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 1150: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 1151: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 1152: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 1153: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 1154: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 1155: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 1156: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 1157: +};
! 1158: +
! 1159: +#ifdef PMD5_ALLOW_SSE2
! 1160: +__attribute__ ((target("sse2"))) MVSTATIC pmd5_status pmd5_init_all(pmd5_context * ctx)
! 1161: +{
! 1162: + int i;
! 1163: + for (i = 0; i < PMD5_SLOTS_MAX; i++) {
! 1164: + ctx->len[i] = 0;
! 1165: + }
! 1166: +
! 1167: + ctx->state_sse2[0] = _mm_set1_epi32(IA);
! 1168: + ctx->state_sse2[1] = _mm_set1_epi32(IB);
! 1169: + ctx->state_sse2[2] = _mm_set1_epi32(IC);
! 1170: + ctx->state_sse2[3] = _mm_set1_epi32(ID);
! 1171: +
! 1172: + return PMD5_SUCCESS;
! 1173: +}
! 1174: +#endif
! 1175: +
! 1176: +#ifdef PMD5_ALLOW_AVX2
! 1177: +__attribute__ ((target("avx2"))) MVSTATIC pmd5_status pmd5_init_all(pmd5_context * ctx)
! 1178: +{
! 1179: + int i;
! 1180: + for (i = 0; i < PMD5_SLOTS_MAX; i++) {
! 1181: + ctx->len[i] = 0;
! 1182: + }
! 1183: +
! 1184: + ctx->state_avx2[0] = _mm256_set1_epi32(IA);
! 1185: + ctx->state_avx2[1] = _mm256_set1_epi32(IB);
! 1186: + ctx->state_avx2[2] = _mm256_set1_epi32(IC);
! 1187: + ctx->state_avx2[3] = _mm256_set1_epi32(ID);
! 1188: +
! 1189: + return PMD5_SUCCESS;
! 1190: +}
! 1191: +#endif
! 1192: +
! 1193: +__attribute__ ((target("default"))) MVSTATIC pmd5_status pmd5_init_all(pmd5_context * ctx)
! 1194: +{
! 1195: + return PMD5_INVALID_SLOT;
! 1196: +}
! 1197: +
! 1198: +#ifdef PMD5_ALLOW_SSE2
! 1199: +__attribute__ ((target("sse2"))) MVSTATIC pmd5_status pmd5_set_slot(pmd5_context * ctx, int slot, uint32_t a, uint32_t b, uint32_t c, uint32_t d)
! 1200: +{
! 1201: + if ((slot >= PMD5_SLOTS_SSE2) || (slot < 0))
! 1202: + return PMD5_INVALID_SLOT;
! 1203: +
! 1204: + __attribute__ ((aligned(32))) uint32_t v[4][PMD5_SLOTS_SSE2];
! 1205: + int i;
! 1206: +
! 1207: + for (i = 0; i < 4; i++) {
! 1208: + _mm_store_si128((__m128i_u*)v[i], ctx->state_sse2[i]);
! 1209: + }
! 1210: +
! 1211: + v[0][slot] = a;
! 1212: + v[1][slot] = b;
! 1213: + v[2][slot] = c;
! 1214: + v[3][slot] = d;
! 1215: +
! 1216: + for (i = 0; i < 4; i++) {
! 1217: + ctx->state_sse2[i] = _mm_loadu_si128((__m128i_u*)v[i]);
! 1218: + }
! 1219: +
! 1220: + return PMD5_SUCCESS;
! 1221: +}
! 1222: +#endif
! 1223: +
! 1224: +#ifdef PMD5_ALLOW_AVX2
! 1225: +__attribute__ ((target("avx2"))) MVSTATIC pmd5_status pmd5_set_slot(pmd5_context * ctx, int slot, uint32_t a, uint32_t b, uint32_t c, uint32_t d)
! 1226: +{
! 1227: + if ((slot >= PMD5_SLOTS_AVX2) || (slot < 0))
! 1228: + return PMD5_INVALID_SLOT;
! 1229: +
! 1230: + __attribute__ ((aligned(32))) uint32_t v[4][PMD5_SLOTS_AVX2];
! 1231: + int i;
! 1232: +
! 1233: + for (i = 0; i < 4; i++) {
! 1234: + _mm256_store_si256((__m256i_u*)v[i], ctx->state_avx2[i]);
! 1235: + }
! 1236: +
! 1237: + v[0][slot] = a;
! 1238: + v[1][slot] = b;
! 1239: + v[2][slot] = c;
! 1240: + v[3][slot] = d;
! 1241: +
! 1242: + for (i = 0; i < 4; i++) {
! 1243: + ctx->state_avx2[i] = _mm256_lddqu_si256((__m256i_u*)v[i]);
! 1244: + }
! 1245: +
! 1246: + return PMD5_SUCCESS;
! 1247: +}
! 1248: +#endif
! 1249: +
! 1250: +__attribute__ ((target("default"))) MVSTATIC pmd5_status pmd5_set_slot(pmd5_context * ctx, int slot, uint32_t a, uint32_t b, uint32_t c, uint32_t d)
! 1251: +{
! 1252: + return PMD5_INVALID_SLOT;
! 1253: +}
! 1254: +
! 1255: +#ifdef PMD5_ALLOW_SSE2
! 1256: +__attribute__ ((target("sse2"))) MVSTATIC pmd5_status pmd5_get_slot(const pmd5_context * ctx, int slot, uint32_t* a, uint32_t* b, uint32_t* c, uint32_t* d)
! 1257: +{
! 1258: + if ((slot >= PMD5_SLOTS_SSE2) || (slot < 0))
! 1259: + return PMD5_INVALID_SLOT;
! 1260: +
! 1261: + __attribute__ ((aligned(32))) uint32_t v[4][PMD5_SLOTS_SSE2];
! 1262: + int i;
! 1263: +
! 1264: + for (i = 0; i < 4; i++) {
! 1265: + _mm_store_si128((__m128i_u*)v[i], ctx->state_sse2[i]);
! 1266: + }
! 1267: +
! 1268: + *a = v[0][slot];
! 1269: + *b = v[1][slot];
! 1270: + *c = v[2][slot];
! 1271: + *d = v[3][slot];
! 1272: +
! 1273: + return PMD5_SUCCESS;
! 1274: +}
! 1275: +#endif
! 1276: +
! 1277: +#ifdef PMD5_ALLOW_AVX2
! 1278: +__attribute__ ((target("avx2"))) MVSTATIC pmd5_status pmd5_get_slot(const pmd5_context * ctx, int slot, uint32_t* a, uint32_t* b, uint32_t* c, uint32_t* d)
! 1279: +{
! 1280: + if ((slot >= PMD5_SLOTS_AVX2) || (slot < 0))
! 1281: + return PMD5_INVALID_SLOT;
! 1282: +
! 1283: + __attribute__ ((aligned(32))) uint32_t v[4][PMD5_SLOTS_AVX2];
! 1284: + int i;
! 1285: +
! 1286: + for (i = 0; i < 4; i++) {
! 1287: + _mm256_store_si256((__m256i_u*)v[i], ctx->state_avx2[i]);
! 1288: + }
! 1289: +
! 1290: + *a = v[0][slot];
! 1291: + *b = v[1][slot];
! 1292: + *c = v[2][slot];
! 1293: + *d = v[3][slot];
! 1294: +
! 1295: + return PMD5_SUCCESS;
! 1296: +}
! 1297: +#endif
! 1298: +
! 1299: +__attribute__ ((target("default"))) MVSTATIC pmd5_status pmd5_get_slot(const pmd5_context * ctx, int slot, uint32_t* a, uint32_t* b, uint32_t* c, uint32_t* d)
! 1300: +{
! 1301: + return PMD5_INVALID_SLOT;
! 1302: +}
! 1303: +
! 1304: +static pmd5_status pmd5_init_slot(pmd5_context * ctx, int slot)
! 1305: +{
! 1306: + return pmd5_set_slot(ctx, slot, IA, IB, IC, ID);
! 1307: +}
! 1308: +
! 1309: +#ifdef PMD5_ALLOW_SSE2
! 1310: +__attribute__ ((target("sse2"))) MVSTATIC void pmd5_process(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX])
! 1311: +{
! 1312: + __m128i W[MD5_DIGEST_LEN], a, b, c, d;
! 1313: +
! 1314: + GET_PMD5_DATA_SSE2(W[ 0], data, 0);
! 1315: + GET_PMD5_DATA_SSE2(W[ 1], data, 4);
! 1316: + GET_PMD5_DATA_SSE2(W[ 2], data, 8);
! 1317: + GET_PMD5_DATA_SSE2(W[ 3], data, 12);
! 1318: + GET_PMD5_DATA_SSE2(W[ 4], data, 16);
! 1319: + GET_PMD5_DATA_SSE2(W[ 5], data, 20);
! 1320: + GET_PMD5_DATA_SSE2(W[ 6], data, 24);
! 1321: + GET_PMD5_DATA_SSE2(W[ 7], data, 28);
! 1322: + GET_PMD5_DATA_SSE2(W[ 8], data, 32);
! 1323: + GET_PMD5_DATA_SSE2(W[ 9], data, 36);
! 1324: + GET_PMD5_DATA_SSE2(W[10], data, 40);
! 1325: + GET_PMD5_DATA_SSE2(W[11], data, 44);
! 1326: + GET_PMD5_DATA_SSE2(W[12], data, 48);
! 1327: + GET_PMD5_DATA_SSE2(W[13], data, 52);
! 1328: + GET_PMD5_DATA_SSE2(W[14], data, 56);
! 1329: + GET_PMD5_DATA_SSE2(W[15], data, 60);
! 1330: +
! 1331: + a = ctx->state_sse2[0];
! 1332: + b = ctx->state_sse2[1];
! 1333: + c = ctx->state_sse2[2];
! 1334: + d = ctx->state_sse2[3];
! 1335: +
! 1336: + SET_SSE2(F, a, b, c, d, W[ 0], S11, 1);
! 1337: + SET_SSE2(F, d, a, b, c, W[ 1], S12, 2);
! 1338: + SET_SSE2(F, c, d, a, b, W[ 2], S13, 3);
! 1339: + SET_SSE2(F, b, c, d, a, W[ 3], S14, 4);
! 1340: + SET_SSE2(F, a, b, c, d, W[ 4], S11, 5);
! 1341: + SET_SSE2(F, d, a, b, c, W[ 5], S12, 6);
! 1342: + SET_SSE2(F, c, d, a, b, W[ 6], S13, 7);
! 1343: + SET_SSE2(F, b, c, d, a, W[ 7], S14, 8);
! 1344: + SET_SSE2(F, a, b, c, d, W[ 8], S11, 9);
! 1345: + SET_SSE2(F, d, a, b, c, W[ 9], S12, 10);
! 1346: + SET_SSE2(F, c, d, a, b, W[10], S13, 11);
! 1347: + SET_SSE2(F, b, c, d, a, W[11], S14, 12);
! 1348: + SET_SSE2(F, a, b, c, d, W[12], S11, 13);
! 1349: + SET_SSE2(F, d, a, b, c, W[13], S12, 14);
! 1350: + SET_SSE2(F, c, d, a, b, W[14], S13, 15);
! 1351: + SET_SSE2(F, b, c, d, a, W[15], S14, 16);
! 1352: +
! 1353: + SET_SSE2(G, a, b, c, d, W[ 1], S21, 17);
! 1354: + SET_SSE2(G, d, a, b, c, W[ 6], S22, 18);
! 1355: + SET_SSE2(G, c, d, a, b, W[11], S23, 19);
! 1356: + SET_SSE2(G, b, c, d, a, W[ 0], S24, 20);
! 1357: + SET_SSE2(G, a, b, c, d, W[ 5], S21, 21);
! 1358: + SET_SSE2(G, d, a, b, c, W[10], S22, 22);
! 1359: + SET_SSE2(G, c, d, a, b, W[15], S23, 23);
! 1360: + SET_SSE2(G, b, c, d, a, W[ 4], S24, 24);
! 1361: + SET_SSE2(G, a, b, c, d, W[ 9], S21, 25);
! 1362: + SET_SSE2(G, d, a, b, c, W[14], S22, 26);
! 1363: + SET_SSE2(G, c, d, a, b, W[ 3], S23, 27);
! 1364: + SET_SSE2(G, b, c, d, a, W[ 8], S24, 28);
! 1365: + SET_SSE2(G, a, b, c, d, W[13], S21, 29);
! 1366: + SET_SSE2(G, d, a, b, c, W[ 2], S22, 30);
! 1367: + SET_SSE2(G, c, d, a, b, W[ 7], S23, 31);
! 1368: + SET_SSE2(G, b, c, d, a, W[12], S24, 32);
! 1369: +
! 1370: + SET_SSE2(H, a, b, c, d, W[ 5], S31, 33);
! 1371: + SET_SSE2(H, d, a, b, c, W[ 8], S32, 34);
! 1372: + SET_SSE2(H, c, d, a, b, W[11], S33, 35);
! 1373: + SET_SSE2(H, b, c, d, a, W[14], S34, 36);
! 1374: + SET_SSE2(H, a, b, c, d, W[ 1], S31, 37);
! 1375: + SET_SSE2(H, d, a, b, c, W[ 4], S32, 38);
! 1376: + SET_SSE2(H, c, d, a, b, W[ 7], S33, 39);
! 1377: + SET_SSE2(H, b, c, d, a, W[10], S34, 40);
! 1378: + SET_SSE2(H, a, b, c, d, W[13], S31, 41);
! 1379: + SET_SSE2(H, d, a, b, c, W[ 0], S32, 42);
! 1380: + SET_SSE2(H, c, d, a, b, W[ 3], S33, 43);
! 1381: + SET_SSE2(H, b, c, d, a, W[ 6], S34, 44);
! 1382: + SET_SSE2(H, a, b, c, d, W[ 9], S31, 45);
! 1383: + SET_SSE2(H, d, a, b, c, W[12], S32, 46);
! 1384: + SET_SSE2(H, c, d, a, b, W[15], S33, 47);
! 1385: + SET_SSE2(H, b, c, d, a, W[ 2], S34, 48);
! 1386: +
! 1387: + SET_SSE2(I, a, b, c, d, W[ 0], S41, 49);
! 1388: + SET_SSE2(I, d, a, b, c, W[ 7], S42, 50);
! 1389: + SET_SSE2(I, c, d, a, b, W[14], S43, 51);
! 1390: + SET_SSE2(I, b, c, d, a, W[ 5], S44, 52);
! 1391: + SET_SSE2(I, a, b, c, d, W[12], S41, 53);
! 1392: + SET_SSE2(I, d, a, b, c, W[ 3], S42, 54);
! 1393: + SET_SSE2(I, c, d, a, b, W[10], S43, 55);
! 1394: + SET_SSE2(I, b, c, d, a, W[ 1], S44, 56);
! 1395: + SET_SSE2(I, a, b, c, d, W[ 8], S41, 57);
! 1396: + SET_SSE2(I, d, a, b, c, W[15], S42, 58);
! 1397: + SET_SSE2(I, c, d, a, b, W[ 6], S43, 59);
! 1398: + SET_SSE2(I, b, c, d, a, W[13], S44, 60);
! 1399: + SET_SSE2(I, a, b, c, d, W[ 4], S41, 61);
! 1400: + SET_SSE2(I, d, a, b, c, W[11], S42, 62);
! 1401: + SET_SSE2(I, c, d, a, b, W[ 2], S43, 63);
! 1402: + SET_SSE2(I, b, c, d, a, W[ 9], S44, 64);
! 1403: +
! 1404: + ctx->state_sse2[0] = _mm_add_epi32(ctx->state_sse2[0], a);
! 1405: + ctx->state_sse2[1] = _mm_add_epi32(ctx->state_sse2[1], b);
! 1406: + ctx->state_sse2[2] = _mm_add_epi32(ctx->state_sse2[2], c);
! 1407: + ctx->state_sse2[3] = _mm_add_epi32(ctx->state_sse2[3], d);
! 1408: +}
! 1409: +#endif
! 1410: +
! 1411: +#ifdef PMD5_ALLOW_AVX2
! 1412: +__attribute__ ((target("avx2"))) MVSTATIC void pmd5_process(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX])
! 1413: +{
! 1414: + __m256i W[MD5_DIGEST_LEN], a, b, c, d;
! 1415: +
! 1416: + GET_PMD5_DATA_AVX2(W[ 0], data, 0);
! 1417: + GET_PMD5_DATA_AVX2(W[ 1], data, 4);
! 1418: + GET_PMD5_DATA_AVX2(W[ 2], data, 8);
! 1419: + GET_PMD5_DATA_AVX2(W[ 3], data, 12);
! 1420: + GET_PMD5_DATA_AVX2(W[ 4], data, 16);
! 1421: + GET_PMD5_DATA_AVX2(W[ 5], data, 20);
! 1422: + GET_PMD5_DATA_AVX2(W[ 6], data, 24);
! 1423: + GET_PMD5_DATA_AVX2(W[ 7], data, 28);
! 1424: + GET_PMD5_DATA_AVX2(W[ 8], data, 32);
! 1425: + GET_PMD5_DATA_AVX2(W[ 9], data, 36);
! 1426: + GET_PMD5_DATA_AVX2(W[10], data, 40);
! 1427: + GET_PMD5_DATA_AVX2(W[11], data, 44);
! 1428: + GET_PMD5_DATA_AVX2(W[12], data, 48);
! 1429: + GET_PMD5_DATA_AVX2(W[13], data, 52);
! 1430: + GET_PMD5_DATA_AVX2(W[14], data, 56);
! 1431: + GET_PMD5_DATA_AVX2(W[15], data, 60);
! 1432: +
! 1433: + a = ctx->state_avx2[0];
! 1434: + b = ctx->state_avx2[1];
! 1435: + c = ctx->state_avx2[2];
! 1436: + d = ctx->state_avx2[3];
! 1437: +
! 1438: + SET_AVX2(F, a, b, c, d, W[ 0], S11, 1);
! 1439: + SET_AVX2(F, d, a, b, c, W[ 1], S12, 2);
! 1440: + SET_AVX2(F, c, d, a, b, W[ 2], S13, 3);
! 1441: + SET_AVX2(F, b, c, d, a, W[ 3], S14, 4);
! 1442: + SET_AVX2(F, a, b, c, d, W[ 4], S11, 5);
! 1443: + SET_AVX2(F, d, a, b, c, W[ 5], S12, 6);
! 1444: + SET_AVX2(F, c, d, a, b, W[ 6], S13, 7);
! 1445: + SET_AVX2(F, b, c, d, a, W[ 7], S14, 8);
! 1446: + SET_AVX2(F, a, b, c, d, W[ 8], S11, 9);
! 1447: + SET_AVX2(F, d, a, b, c, W[ 9], S12, 10);
! 1448: + SET_AVX2(F, c, d, a, b, W[10], S13, 11);
! 1449: + SET_AVX2(F, b, c, d, a, W[11], S14, 12);
! 1450: + SET_AVX2(F, a, b, c, d, W[12], S11, 13);
! 1451: + SET_AVX2(F, d, a, b, c, W[13], S12, 14);
! 1452: + SET_AVX2(F, c, d, a, b, W[14], S13, 15);
! 1453: + SET_AVX2(F, b, c, d, a, W[15], S14, 16);
! 1454: +
! 1455: + SET_AVX2(G, a, b, c, d, W[ 1], S21, 17);
! 1456: + SET_AVX2(G, d, a, b, c, W[ 6], S22, 18);
! 1457: + SET_AVX2(G, c, d, a, b, W[11], S23, 19);
! 1458: + SET_AVX2(G, b, c, d, a, W[ 0], S24, 20);
! 1459: + SET_AVX2(G, a, b, c, d, W[ 5], S21, 21);
! 1460: + SET_AVX2(G, d, a, b, c, W[10], S22, 22);
! 1461: + SET_AVX2(G, c, d, a, b, W[15], S23, 23);
! 1462: + SET_AVX2(G, b, c, d, a, W[ 4], S24, 24);
! 1463: + SET_AVX2(G, a, b, c, d, W[ 9], S21, 25);
! 1464: + SET_AVX2(G, d, a, b, c, W[14], S22, 26);
! 1465: + SET_AVX2(G, c, d, a, b, W[ 3], S23, 27);
! 1466: + SET_AVX2(G, b, c, d, a, W[ 8], S24, 28);
! 1467: + SET_AVX2(G, a, b, c, d, W[13], S21, 29);
! 1468: + SET_AVX2(G, d, a, b, c, W[ 2], S22, 30);
! 1469: + SET_AVX2(G, c, d, a, b, W[ 7], S23, 31);
! 1470: + SET_AVX2(G, b, c, d, a, W[12], S24, 32);
! 1471: +
! 1472: + SET_AVX2(H, a, b, c, d, W[ 5], S31, 33);
! 1473: + SET_AVX2(H, d, a, b, c, W[ 8], S32, 34);
! 1474: + SET_AVX2(H, c, d, a, b, W[11], S33, 35);
! 1475: + SET_AVX2(H, b, c, d, a, W[14], S34, 36);
! 1476: + SET_AVX2(H, a, b, c, d, W[ 1], S31, 37);
! 1477: + SET_AVX2(H, d, a, b, c, W[ 4], S32, 38);
! 1478: + SET_AVX2(H, c, d, a, b, W[ 7], S33, 39);
! 1479: + SET_AVX2(H, b, c, d, a, W[10], S34, 40);
! 1480: + SET_AVX2(H, a, b, c, d, W[13], S31, 41);
! 1481: + SET_AVX2(H, d, a, b, c, W[ 0], S32, 42);
! 1482: + SET_AVX2(H, c, d, a, b, W[ 3], S33, 43);
! 1483: + SET_AVX2(H, b, c, d, a, W[ 6], S34, 44);
! 1484: + SET_AVX2(H, a, b, c, d, W[ 9], S31, 45);
! 1485: + SET_AVX2(H, d, a, b, c, W[12], S32, 46);
! 1486: + SET_AVX2(H, c, d, a, b, W[15], S33, 47);
! 1487: + SET_AVX2(H, b, c, d, a, W[ 2], S34, 48);
! 1488: +
! 1489: + SET_AVX2(I, a, b, c, d, W[ 0], S41, 49);
! 1490: + SET_AVX2(I, d, a, b, c, W[ 7], S42, 50);
! 1491: + SET_AVX2(I, c, d, a, b, W[14], S43, 51);
! 1492: + SET_AVX2(I, b, c, d, a, W[ 5], S44, 52);
! 1493: + SET_AVX2(I, a, b, c, d, W[12], S41, 53);
! 1494: + SET_AVX2(I, d, a, b, c, W[ 3], S42, 54);
! 1495: + SET_AVX2(I, c, d, a, b, W[10], S43, 55);
! 1496: + SET_AVX2(I, b, c, d, a, W[ 1], S44, 56);
! 1497: + SET_AVX2(I, a, b, c, d, W[ 8], S41, 57);
! 1498: + SET_AVX2(I, d, a, b, c, W[15], S42, 58);
! 1499: + SET_AVX2(I, c, d, a, b, W[ 6], S43, 59);
! 1500: + SET_AVX2(I, b, c, d, a, W[13], S44, 60);
! 1501: + SET_AVX2(I, a, b, c, d, W[ 4], S41, 61);
! 1502: + SET_AVX2(I, d, a, b, c, W[11], S42, 62);
! 1503: + SET_AVX2(I, c, d, a, b, W[ 2], S43, 63);
! 1504: + SET_AVX2(I, b, c, d, a, W[ 9], S44, 64);
! 1505: +
! 1506: + ctx->state_avx2[0] = _mm256_add_epi32(ctx->state_avx2[0], a);
! 1507: + ctx->state_avx2[1] = _mm256_add_epi32(ctx->state_avx2[1], b);
! 1508: + ctx->state_avx2[2] = _mm256_add_epi32(ctx->state_avx2[2], c);
! 1509: + ctx->state_avx2[3] = _mm256_add_epi32(ctx->state_avx2[3], d);
! 1510: +}
! 1511: +#endif
! 1512: +
! 1513: +__attribute__ ((target("default"))) MVSTATIC void pmd5_process(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX])
! 1514: +{
! 1515: +}
! 1516: +
! 1517: +static pmd5_status pmd5_update_all_simple(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX], uint64_t length, uint64_t stride)
! 1518: +{
! 1519: + const uint8_t * ptrs[PMD5_SLOTS_MAX];
! 1520: +
! 1521: + if (!length) return PMD5_SUCCESS;
! 1522: +
! 1523: + int slots = pmd5_slots();
! 1524: +
! 1525: + if (!stride) stride = 64;
! 1526: +
! 1527: + int i;
! 1528: + for (i = 0; i < slots; i++) {
! 1529: + ptrs[i] = data[i];
! 1530: + ctx->len[i] += length;
! 1531: + if (!ptrs[i]) ptrs[i] = md5_padding;
! 1532: + }
! 1533: +
! 1534: + while (length >= 64) {
! 1535: + pmd5_process(ctx, ptrs);
! 1536: + length -= 64;
! 1537: + for (i = 0; i < slots; i++) {
! 1538: + if (data[i]) ptrs[i] += stride;
! 1539: + }
! 1540: + }
! 1541: +
! 1542: + if (length) return PMD5_UNALIGNED_UPDATE;
! 1543: +
! 1544: + for (i = 0; i < slots; i++) {
! 1545: + if (data[i]) data[i] = ptrs[i];
! 1546: + }
! 1547: +
! 1548: + return PMD5_SUCCESS;
! 1549: +}
! 1550: +
! 1551: +static pmd5_status pmd5_update_all(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX], uint64_t lengths[PMD5_SLOTS_MAX])
! 1552: +{
! 1553: + uint64_t length = 0;
! 1554: + int slots = pmd5_slots();
! 1555: +
! 1556: + int i;
! 1557: + for (i = 0; i < slots; i++) {
! 1558: + if ((length == 0) || (lengths[i] < length)) length = lengths[i];
! 1559: + }
! 1560: +
! 1561: + for (i = 0; i < slots; i++) {
! 1562: + lengths[i] -= length;
! 1563: + }
! 1564: +
! 1565: + return pmd5_update_all_simple(ctx, data, length, 0);
! 1566: +}
! 1567: +
! 1568: +static pmd5_status pmd5_finish_slot_with_extra(pmd5_context * pctx, uint8_t digest[MD5_DIGEST_LEN], int slot, const uint8_t * data, uint64_t length)
! 1569: +{
! 1570: + MD5_CTX ctx;
! 1571: +
! 1572: + if ((slot >= pmd5_slots()) || (slot < 0))
! 1573: + return PMD5_INVALID_SLOT;
! 1574: +
! 1575: + pmd5_to_md5(pctx, &ctx, slot);
! 1576: + if (data && length) {
! 1577: + MD5_Update(&ctx, data, length);
! 1578: + }
! 1579: + MD5_Final(digest, &ctx);
! 1580: +
! 1581: + return PMD5_SUCCESS;
! 1582: +}
! 1583: +
! 1584: +static pmd5_status pmd5_finish_slot(pmd5_context * pctx, uint8_t digest[MD5_DIGEST_LEN], int slot)
! 1585: +{
! 1586: + return pmd5_finish_slot_with_extra(pctx, digest, slot, NULL, 0);
! 1587: +}
! 1588: +
! 1589: +static pmd5_status pmd5_finish_all(pmd5_context * ctx, uint8_t digests[PMD5_SLOTS_MAX][MD5_DIGEST_LEN])
! 1590: +{
! 1591: + int i;
! 1592: + for (i = 0; i < pmd5_slots(); i++) {
! 1593: + pmd5_finish_slot_with_extra(ctx, digests[i], i, NULL, 0);
! 1594: + }
! 1595: + return PMD5_SUCCESS;
! 1596: +}
! 1597: +
! 1598: +static pmd5_status md5_to_pmd5(const MD5_CTX * ctx, pmd5_context * pctx, int slot)
! 1599: +{
! 1600: + if ((slot >= pmd5_slots()) || (slot < 0))
! 1601: + return PMD5_INVALID_SLOT;
! 1602: +
! 1603: + // TODO This function ignores buffered but as of yet unhashed data. We're not using this function, just noting.
! 1604: +
! 1605: +#ifdef USE_OPENSSL
! 1606: + pctx->len[slot] = (ctx->Nl >> 3) + ((uint64_t)ctx->Nh << 29);
! 1607: +#else
! 1608: + pctx->len[slot] = ctx->totalN + ((uint64_t)ctx->totalN2 << 32);
! 1609: +#endif
! 1610: + return pmd5_set_slot(pctx, slot, (uint32_t)ctx->A, (uint32_t)ctx->B, (uint32_t)ctx->C, (uint32_t)ctx->D);
! 1611: +}
! 1612: +
! 1613: +static pmd5_status pmd5_to_md5(const pmd5_context * pctx, MD5_CTX * ctx, int slot)
! 1614: +{
! 1615: + if ((slot >= pmd5_slots()) || (slot < 0))
! 1616: + return PMD5_INVALID_SLOT;
! 1617: +
! 1618: + MD5_Init(ctx);
! 1619: +
! 1620: +#ifdef USE_OPENSSL
! 1621: + ctx->Nl = (pctx->len[slot] << 3) & 0xFFFFFFFF;
! 1622: + ctx->Nh = pctx->len[slot] >> 29;
! 1623: +
! 1624: + uint32_t a, b, c, d;
! 1625: + pmd5_status ret = pmd5_get_slot(pctx, slot, &a, &b, &c, &d);
! 1626: + if (ret == PMD5_SUCCESS) {
! 1627: + ctx->A = a;
! 1628: + ctx->B = b;
! 1629: + ctx->C = c;
! 1630: + ctx->D = d;
! 1631: + }
! 1632: + return ret;
! 1633: +#else
! 1634: + ctx->totalN = pctx->len[slot] & 0xFFFFFFFF;
! 1635: + ctx->totalN2 = pctx->len[slot] >> 32;
! 1636: + return pmd5_get_slot(pctx, slot, &ctx->A, &ctx->B, &ctx->C, &ctx->D);
! 1637: +#endif
! 1638: +}
! 1639: +
! 1640: +/* With GCC 10 putting these implementations inside 'extern "C"' causes an
! 1641: + assembler error. That worked fine on GCC 5-9 and clang 6-10...
! 1642: + */
! 1643: +
! 1644: +static inline int md5_parallel_slots_cpp()
! 1645: +{
! 1646: + int slots = pmd5_slots();
! 1647: + if (slots == 0) return 1;
! 1648: + return slots;
! 1649: +}
! 1650: +
! 1651: +static inline int md5_parallel_cpp(int streams, char** buf, int* len, char** sum, char* pre4, char* post4)
! 1652: +{
! 1653: + int slots = md5_parallel_slots_cpp();
! 1654: + if ((streams < 1) || (streams > slots)) return 0;
! 1655: + if (pre4 && post4) return 0;
! 1656: +
! 1657: + if (slots == 1) {
! 1658: + MD5_CTX ctx;
! 1659: + MD5_Init(&ctx);
! 1660: + if (pre4) {
! 1661: + MD5_Update(&ctx, (const unsigned char*)pre4, 4);
! 1662: + }
! 1663: + MD5_Update(&ctx, (const unsigned char*)buf[0], len[0]);
! 1664: + if (post4) {
! 1665: + MD5_Update(&ctx, (const unsigned char*)post4, 4);
! 1666: + }
! 1667: + if (sum[0]) {
! 1668: + MD5_Final((uint8_t*)sum[0], &ctx);
! 1669: + }
! 1670: + return 0;
! 1671: + }
! 1672: +
! 1673: + int i;
! 1674: + int active[PMD5_SLOTS_MAX];
! 1675: + char* buffers[PMD5_SLOTS_MAX];
! 1676: + uint64_t left[PMD5_SLOTS_MAX];
! 1677: + for (i = 0; i < PMD5_SLOTS_MAX; i++) {
! 1678: + active[i] = streams > i;
! 1679: + if (i < streams) {
! 1680: + buffers[i] = buf[i];
! 1681: + left[i] = (uint64_t)len[i];
! 1682: + } else {
! 1683: + buffers[i] = NULL;
! 1684: + left[i] = 0;
! 1685: + }
! 1686: + }
! 1687: + MD5_CTX results[PMD5_SLOTS_MAX];
! 1688: +
! 1689: + pmd5_context ctx_simd;
! 1690: + if (pmd5_init_all(&ctx_simd) != PMD5_SUCCESS) return 0;
! 1691: +
! 1692: + if (pre4) {
! 1693: + char temp_buffers[PMD5_SLOTS_MAX][64];
! 1694: + int have_any = 0;
! 1695: + for (i = 0; i < slots; i++) {
! 1696: + if (active[i]) {
! 1697: + if (left[i] < 60) {
! 1698: + MD5_Init(&results[i]);
! 1699: + MD5_Update(&results[i], (const unsigned char*)pre4, 4);
! 1700: + MD5_Update(&results[i], (const unsigned char*)buf[i], left[i]);
! 1701: + active[i] = 0;
! 1702: + left[i] = 0;
! 1703: + } else {
! 1704: + memcpy(temp_buffers[i], pre4, 4);
! 1705: + memcpy(temp_buffers[i] + 4, buffers[i], 60);
! 1706: + buffers[i] += 60;
! 1707: + left[i] -= 60;
! 1708: + have_any = 1;
! 1709: + }
! 1710: + }
! 1711: + }
! 1712: +
! 1713: + if (have_any) {
! 1714: + char* ptrs[PMD5_SLOTS_MAX];
! 1715: + for (i = 0; i < PMD5_SLOTS_MAX; i++) {
! 1716: + ptrs[i] = &temp_buffers[i][0];
! 1717: + }
! 1718: + if (pmd5_update_all_simple(&ctx_simd, (const uint8_t**)ptrs, 64, 0) != PMD5_SUCCESS) {
! 1719: + return 0;
! 1720: + }
! 1721: + }
! 1722: + }
! 1723: +
! 1724: + int failed = 0;
! 1725: + while (true) {
! 1726: + for (i = 0; i < slots; i++) {
! 1727: + if (active[i] && (left[i] < 64)) {
! 1728: + if (pmd5_to_md5(&ctx_simd, &results[i], i) != PMD5_SUCCESS) {
! 1729: + failed = 1;
! 1730: + }
! 1731: + active[i] = 0;
! 1732: + }
! 1733: + }
! 1734: +
! 1735: + uint64_t shortest = 0;
! 1736: + for (i = 0; i < slots; i++) {
! 1737: + if (!active[i]) {
! 1738: + buffers[i] = NULL;
! 1739: + } else if ((shortest == 0) || (left[i] < shortest)) {
! 1740: + shortest = left[i];
! 1741: + }
! 1742: + }
! 1743: +
! 1744: + if (shortest > 0) {
! 1745: + shortest = shortest & ~63;
! 1746: + if (pmd5_update_all_simple(&ctx_simd, (const uint8_t**)buffers, shortest, 0) != PMD5_SUCCESS) {
! 1747: + failed = 1;
! 1748: + }
! 1749: + for (i = 0; i < slots; i++) {
! 1750: + if (active[i]) {
! 1751: + left[i] -= shortest;
! 1752: + }
! 1753: + }
! 1754: + }
! 1755: +
! 1756: + if (failed) {
! 1757: + return 0;
! 1758: + } else {
! 1759: + int have_any = 0;
! 1760: + for (i = 0; i < slots; i++) {
! 1761: + have_any |= active[i];
! 1762: + }
! 1763: + if (!have_any) {
! 1764: + break;
! 1765: + }
! 1766: + }
! 1767: + }
! 1768: +
! 1769: + for (i = 0; i < slots; i++) {
! 1770: + if (i < streams) {
! 1771: + if (left[i] > 0) {
! 1772: + // buffer[i] == NULL here
! 1773: + MD5_Update(&results[i], (const unsigned char*)buf[i] + len[i] - left[i], left[i]);
! 1774: + }
! 1775: + if (post4) {
! 1776: + MD5_Update(&results[i], (const unsigned char*)post4, 4);
! 1777: + }
! 1778: + if (sum[i]) {
! 1779: + MD5_Final((uint8_t*)sum[i], &results[i]);
! 1780: + }
! 1781: + }
! 1782: + }
! 1783: +
! 1784: + return 1;
! 1785: +}
! 1786: +
! 1787: +// each pmd5_context needs to be 32-byte aligned
! 1788: +#define MD5P8_Contexts_simd(ctx, index) ((pmd5_context*)((((uintptr_t)((ctx)->context_storage) + 31) & ~31) + (index)*((sizeof(pmd5_context) + 31) & ~31)))
! 1789: +
! 1790: +static inline void MD5P8_Init_cpp(MD5P8_CTX *ctx)
! 1791: +{
! 1792: + int i;
! 1793: + for (i = 0; i < (pmd5_slots() == PMD5_SLOTS_AVX2 ? 1 : 2); i++) {
! 1794: + pmd5_init_all(MD5P8_Contexts_simd(ctx, i));
! 1795: + }
! 1796: + ctx->used = 0;
! 1797: + ctx->next = 0;
! 1798: +}
! 1799: +
! 1800: +static inline void MD5P8_Update_cpp(MD5P8_CTX *ctx, const uchar *input, uint32 length)
! 1801: +{
! 1802: + int slots = pmd5_slots();
! 1803: + uint32 pos = 0;
! 1804: +
! 1805: + if ((ctx->used) || (length < 512)) {
! 1806: + int cpy = MIN(length, 512 - ctx->used);
! 1807: + memcpy(&ctx->buffer[ctx->used], input, cpy);
! 1808: + ctx->used += cpy;
! 1809: + length -= cpy;
! 1810: + pos += cpy;
! 1811: +
! 1812: + if (ctx->used == 512) {
! 1813: + if (slots == PMD5_SLOTS_AVX2) {
! 1814: + const uint8_t* ptrs[PMD5_SLOTS_MAX] = {
! 1815: + (uint8_t*)ctx->buffer,
! 1816: + (uint8_t*)(ctx->buffer + 64),
! 1817: + (uint8_t*)(ctx->buffer + 128),
! 1818: + (uint8_t*)(ctx->buffer + 192),
! 1819: + (uint8_t*)(ctx->buffer + 256),
! 1820: + (uint8_t*)(ctx->buffer + 320),
! 1821: + (uint8_t*)(ctx->buffer + 384),
! 1822: + (uint8_t*)(ctx->buffer + 448)
! 1823: + };
! 1824: + pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 0), ptrs, 64, 0);
! 1825: + } else {
! 1826: + const uint8_t* ptrs1[PMD5_SLOTS_MAX] = {
! 1827: + (uint8_t*)ctx->buffer,
! 1828: + (uint8_t*)(ctx->buffer + 64),
! 1829: + (uint8_t*)(ctx->buffer + 128),
! 1830: + (uint8_t*)(ctx->buffer + 192)
! 1831: + };
! 1832: + const uint8_t* ptrs2[PMD5_SLOTS_MAX] = {
! 1833: + (uint8_t*)(ctx->buffer + 256),
! 1834: + (uint8_t*)(ctx->buffer + 320),
! 1835: + (uint8_t*)(ctx->buffer + 384),
! 1836: + (uint8_t*)(ctx->buffer + 448)
! 1837: + };
! 1838: + pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 0), ptrs1, 64, 0);
! 1839: + pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 1), ptrs2, 64, 0);
! 1840: + }
! 1841: + ctx->used = 0;
! 1842: + }
! 1843: + }
! 1844: +
! 1845: + if (length >= 512) {
! 1846: + uint32 blocks = length / 512;
! 1847: + if (slots == PMD5_SLOTS_AVX2) {
! 1848: + const uint8_t* ptrs[8] = {
! 1849: + (uint8_t*)(input + pos),
! 1850: + (uint8_t*)(input + pos + 64),
! 1851: + (uint8_t*)(input + pos + 128),
! 1852: + (uint8_t*)(input + pos + 192),
! 1853: + (uint8_t*)(input + pos + 256),
! 1854: + (uint8_t*)(input + pos + 320),
! 1855: + (uint8_t*)(input + pos + 384),
! 1856: + (uint8_t*)(input + pos + 448)
! 1857: + };
! 1858: + pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 0), ptrs, blocks * 64, 512);
! 1859: + } else {
! 1860: + const uint8_t* ptrs1[4] = {
! 1861: + (uint8_t*)(input + pos),
! 1862: + (uint8_t*)(input + pos + 64),
! 1863: + (uint8_t*)(input + pos + 128),
! 1864: + (uint8_t*)(input + pos + 192)
! 1865: + };
! 1866: + const uint8_t* ptrs2[4] = {
! 1867: + (uint8_t*)(input + pos + 256),
! 1868: + (uint8_t*)(input + pos + 320),
! 1869: + (uint8_t*)(input + pos + 384),
! 1870: + (uint8_t*)(input + pos + 448)
! 1871: + };
! 1872: + pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 0), ptrs1, blocks * 64, 512);
! 1873: + pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 1), ptrs2, blocks * 64, 512);
! 1874: + }
! 1875: + pos += blocks * 512;
! 1876: + length -= blocks * 512;
! 1877: + }
! 1878: +
! 1879: + if (length) {
! 1880: + memcpy(ctx->buffer, &input[pos], length);
! 1881: + ctx->used = length;
! 1882: + }
! 1883: +}
! 1884: +
! 1885: +static inline void MD5P8_Final_cpp(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx)
! 1886: +{
! 1887: + int i;
! 1888: + uint32 low = 0, high = 0, sub = ctx->used ? 512 - ctx->used : 0;
! 1889: + if (ctx->used) {
! 1890: + uchar tmp[512];
! 1891: + memset(tmp, 0, 512);
! 1892: + MD5P8_Update(ctx, tmp, 512 - ctx->used);
! 1893: + }
! 1894: +
! 1895: + uchar state[34*4] = {0};
! 1896: +
! 1897: + MD5_CTX tmp;
! 1898: + for (i = 0; i < 8; i++) {
! 1899: + if (pmd5_slots() == PMD5_SLOTS_AVX2) {
! 1900: + pmd5_to_md5(MD5P8_Contexts_simd(ctx, 0), &tmp, i);
! 1901: + } else if (i < 4) {
! 1902: + pmd5_to_md5(MD5P8_Contexts_simd(ctx, 0), &tmp, i);
! 1903: + } else {
! 1904: + pmd5_to_md5(MD5P8_Contexts_simd(ctx, 1), &tmp, i - 4);
! 1905: + }
! 1906: +#ifdef USE_OPENSSL
! 1907: + if (low + tmp.Nl < low) high++;
! 1908: + low += tmp.Nl;
! 1909: + high += tmp.Nh;
! 1910: +#else
! 1911: + if (low + tmp.totalN < low) high++;
! 1912: + low += tmp.totalN;
! 1913: + high += tmp.totalN2;
! 1914: +#endif
! 1915: + SIVALu(state, i*16, tmp.A);
! 1916: + SIVALu(state, i*16 + 4, tmp.B);
! 1917: + SIVALu(state, i*16 + 8, tmp.C);
! 1918: + SIVALu(state, i*16 + 12, tmp.D);
! 1919: + }
! 1920: +
! 1921: +#ifndef USE_OPENSSL
! 1922: + high = (low >> 29) | (high << 3);
! 1923: + low = (low << 3);
! 1924: +#endif
! 1925: +
! 1926: + sub <<= 3;
! 1927: + if (low - sub > low) high--;
! 1928: + low -= sub;
! 1929: +
! 1930: + SIVALu(state, 32*4, low);
! 1931: + SIVALu(state, 33*4, high);
! 1932: +
! 1933: + MD5_CTX md;
! 1934: + MD5_Init(&md);
! 1935: + MD5_Update(&md, state, 34*4);
! 1936: + MD5_Final(digest, &md);
! 1937: +}
! 1938: +
! 1939: +extern "C" {
! 1940: +
! 1941: +int md5_parallel_slots()
! 1942: +{
! 1943: + return md5_parallel_slots_cpp();
! 1944: +}
! 1945: +
! 1946: +int md5_parallel(int streams, char** buf, int* len, char** sum, char* pre4, char* post4)
! 1947: +{
! 1948: + return md5_parallel_cpp(streams, buf, len, sum, pre4, post4);
! 1949: +}
! 1950: +
! 1951: +void MD5P8_Init(MD5P8_CTX *ctx)
! 1952: +{
! 1953: + MD5P8_Init_cpp(ctx);
! 1954: +}
! 1955: +
! 1956: +void MD5P8_Update(MD5P8_CTX *ctx, const uchar *input, uint32 length)
! 1957: +{
! 1958: + MD5P8_Update_cpp(ctx, input, length);
! 1959: +}
! 1960: +
! 1961: +void MD5P8_Final(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx)
! 1962: +{
! 1963: + MD5P8_Final_cpp(digest, ctx);
! 1964: +}
! 1965: +
! 1966: +} // "C"
! 1967: +
! 1968: +#endif /* HAVE_SIMD */
! 1969: +#endif /* __cplusplus */
! 1970: +#endif /* __x86_64__ */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to md5p8.diff CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / rsync / patches