Annotation of embedaddon/rsync/simd-md5-parallel-x86_64.cpp, revision 1.1
1.1 ! misho 1: /*
! 2: * SSE2/AVX2-optimized routines to process multiple MD5 streams in parallel.
! 3: *
! 4: * Original author: Nicolas Noble, 2017
! 5: * Modifications: Jorrit Jongma, 2020
! 6: *
! 7: * The original code was released in the public domain by the original author,
! 8: * falling back to the MIT license ( http://www.opensource.org/licenses/MIT )
! 9: * in case public domain does not apply in your country. These modifications
! 10: * are likewise released in the public domain, with the same MIT license
! 11: * fallback.
! 12: *
! 13: * The original publication can be found at:
! 14: *
! 15: * https://github.com/nicolasnoble/sse-hash
! 16: */
! 17: /*
! 18: * Nicolas' original code has been extended to add AVX2 support, all non-SIMD
! 19: * MD5 code has been removed and those code paths rerouted to use the MD5
! 20: * code already present in rsync, and wrapper functions have been added. The
! 21: * MD5P8 code is also new, and is the reason for the new stride parameter.
! 22: *
! 23: * This code allows multiple independent MD5 streams to be processed in
! 24: * parallel, 4 with SSE2, 8 with AVX2. While single-stream performance is
! 25: * lower than that of the original C routines for MD5, the processing of
! 26: * additional streams is "for free".
! 27: *
! 28: * Single streams are rerouted to rsync's normal MD5 code as that is faster
! 29: * for that case. A further optimization is possible by using SSE2 code on
! 30: * AVX2-supporting CPUs when the number of streams is 2, 3, or 4. This is not
! 31: * implemented here as it would require some restructuring, and in practise
! 32: * the code here is only rarely called with less than the maximum amount of
! 33: * streams (typically once at the end of each checksum2'd file).
! 34: *
! 35: * Benchmarks (in MB/s) C ASM SSE2*1 SSE2*4 AVX2*1 AVX2*8
! 36: * - Intel Atom D2700 302 334 166 664 N/A N/A
! 37: * - Intel i7-7700hq 351 376 289 1156 273 2184
! 38: * - AMD ThreadRipper 2950x 728 784 568 2272 430 3440
! 39: */
! 40:
! 41: #ifdef __x86_64__
! 42: #ifdef __cplusplus
! 43:
! 44: extern "C" {
! 45:
! 46: #include "rsync.h"
! 47:
! 48: }
! 49:
! 50: #ifdef HAVE_SIMD
! 51:
! 52: #ifndef BENCHMARK_SIMD_CHECKSUM2
! 53: #define PMD5_ALLOW_SSE2 // debugging
! 54: #define PMD5_ALLOW_AVX2 // debugging
! 55: #endif
! 56:
! 57: #ifdef PMD5_ALLOW_AVX2
! 58: #ifndef PMD5_ALLOW_SSE2
! 59: #define PMD5_ALLOW_SSE2
! 60: #endif
! 61: #endif
! 62:
! 63: #include <stdint.h>
! 64: #include <string.h>
! 65:
! 66: #include <immintrin.h>
! 67:
! 68: /* Some clang versions don't like it when you use static with multi-versioned functions: linker errors */
! 69: #ifdef __clang__
! 70: #define MVSTATIC
! 71: #else
! 72: #define MVSTATIC static
! 73: #endif
! 74:
! 75: // Missing from the headers on gcc 6 and older, clang 8 and older
! 76: typedef long long __m128i_u __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));
! 77: typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, __aligned__(1)));
! 78:
! 79: #define PMD5_SLOTS_DEFAULT 0
! 80: #define PMD5_SLOTS_SSE2 4
! 81: #define PMD5_SLOTS_AVX2 8
! 82: #define PMD5_SLOTS_MAX PMD5_SLOTS_AVX2
! 83:
! 84: #ifdef PMD5_ALLOW_SSE2
! 85: __attribute__ ((target("sse2"))) MVSTATIC int pmd5_slots()
! 86: {
! 87: return PMD5_SLOTS_SSE2;
! 88: }
! 89: #endif
! 90:
! 91: #ifdef PMD5_ALLOW_AVX2
! 92: __attribute__ ((target("avx2"))) MVSTATIC int pmd5_slots()
! 93: {
! 94: return PMD5_SLOTS_AVX2;
! 95: }
! 96: #endif
! 97:
! 98: __attribute__ ((target("default"))) MVSTATIC int pmd5_slots()
! 99: {
! 100: return PMD5_SLOTS_DEFAULT;
! 101: }
! 102:
! 103: /* The parallel MD5 context structure. */
! 104: typedef struct {
! 105: __m128i state_sse2[4];
! 106: __m256i state_avx2[4];
! 107: uint64_t len[PMD5_SLOTS_MAX];
! 108: } pmd5_context;
! 109:
! 110: /* The status returned by the various functions below. */
! 111: typedef enum {
! 112: PMD5_SUCCESS,
! 113: PMD5_INVALID_SLOT,
! 114: PMD5_UNALIGNED_UPDATE,
! 115: } pmd5_status;
! 116:
! 117: /* Initializes all slots in the given pmd5 context. */
! 118: __attribute__ ((target("default"))) MVSTATIC pmd5_status pmd5_init_all(pmd5_context * ctx);
! 119:
! 120: /* Initializes a single slot out in the given pmd5 context. */
! 121: static pmd5_status pmd5_init_slot(pmd5_context * ctx, int slot);
! 122:
! 123: /* Makes an MD5 update on all slots in parallel, given the same exact length on all streams.
! 124: The stream pointers will be incremented accordingly.
! 125: It is valid for a stream pointer to be NULL. Garbage will then be hashed into its corresponding slot.
! 126: The argument length NEEDS to be a multiple of 64. If not, an error is returned, and the context is corrupted.
! 127: Stride defaults to 64 if 0 is passed. */
! 128: static pmd5_status pmd5_update_all_simple(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX], uint64_t length, uint64_t stride);
! 129:
! 130: /* Makes an MD5 update on all slots in parallel, given different lengths.
! 131: The stream pointers will be incremented accordingly.
! 132: The lengths will be decreased accordingly. Not all data might be consumed.
! 133: It is valid for a stream pointer to be NULL. Garbage will then be hashed into its corresponding slot.
! 134: The argument lengths NEEDS to contain only multiples of 64. If not, an error is returned, and the context is corrupted. */
! 135: static pmd5_status pmd5_update_all(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX], uint64_t lengths[PMD5_SLOTS_MAX]);
! 136:
! 137: /* Finishes all slots at once. Fills in all digests. */
! 138: static pmd5_status pmd5_finish_all(pmd5_context * ctx, uint8_t digests[PMD5_SLOTS_MAX][MD5_DIGEST_LEN]);
! 139:
! 140: /* Finishes one slot. The other slots will be unnaffected. The finished slot can then continue to hash garbage using
! 141: a NULL pointer as its stream argument, or needs to be reinitialized using pmd5_init_slot before being usable again. */
! 142: static pmd5_status pmd5_finish_slot(pmd5_context * ctx, uint8_t digest[MD5_DIGEST_LEN], int slot);
! 143:
! 144: /* Finishes one slot. Extra data is allowed to be passed on as an argument. Length DOESN'T need to be a
! 145: multiple of 64. The other slots will be unaffected. The finished slot can then continue to hash garbage using
! 146: a NULL pointer as its stream argument, or needs to be reinitialized using pmd5_init_slot before being usable again. */
! 147: static pmd5_status pmd5_finish_slot_with_extra(pmd5_context * ctx, uint8_t digest[MD5_DIGEST_LEN], int slot, const uint8_t * data, uint64_t length);
! 148:
! 149: /* Insert a normal MD5 context into a given slot of a given parallel MD5 context. */
! 150: static pmd5_status md5_to_pmd5(const MD5_CTX * ctx, pmd5_context * pctx, int slot);
! 151:
! 152: /* Extract a normal MD5 context from a given slot of a given parallel MD5 context. */
! 153: static pmd5_status pmd5_to_md5(const pmd5_context * pctx, MD5_CTX * ctx, int slot);
! 154:
! 155: #define S11 7
! 156: #define S12 12
! 157: #define S13 17
! 158: #define S14 22
! 159: #define S21 5
! 160: #define S22 9
! 161: #define S23 14
! 162: #define S24 20
! 163: #define S31 4
! 164: #define S32 11
! 165: #define S33 16
! 166: #define S34 23
! 167: #define S41 6
! 168: #define S42 10
! 169: #define S43 15
! 170: #define S44 21
! 171:
! 172: #define T1 0xD76AA478
! 173: #define T2 0xE8C7B756
! 174: #define T3 0x242070DB
! 175: #define T4 0xC1BDCEEE
! 176: #define T5 0xF57C0FAF
! 177: #define T6 0x4787C62A
! 178: #define T7 0xA8304613
! 179: #define T8 0xFD469501
! 180: #define T9 0x698098D8
! 181: #define T10 0x8B44F7AF
! 182: #define T11 0xFFFF5BB1
! 183: #define T12 0x895CD7BE
! 184: #define T13 0x6B901122
! 185: #define T14 0xFD987193
! 186: #define T15 0xA679438E
! 187: #define T16 0x49B40821
! 188: #define T17 0xF61E2562
! 189: #define T18 0xC040B340
! 190: #define T19 0x265E5A51
! 191: #define T20 0xE9B6C7AA
! 192: #define T21 0xD62F105D
! 193: #define T22 0x02441453
! 194: #define T23 0xD8A1E681
! 195: #define T24 0xE7D3FBC8
! 196: #define T25 0x21E1CDE6
! 197: #define T26 0xC33707D6
! 198: #define T27 0xF4D50D87
! 199: #define T28 0x455A14ED
! 200: #define T29 0xA9E3E905
! 201: #define T30 0xFCEFA3F8
! 202: #define T31 0x676F02D9
! 203: #define T32 0x8D2A4C8A
! 204: #define T33 0xFFFA3942
! 205: #define T34 0x8771F681
! 206: #define T35 0x6D9D6122
! 207: #define T36 0xFDE5380C
! 208: #define T37 0xA4BEEA44
! 209: #define T38 0x4BDECFA9
! 210: #define T39 0xF6BB4B60
! 211: #define T40 0xBEBFBC70
! 212: #define T41 0x289B7EC6
! 213: #define T42 0xEAA127FA
! 214: #define T43 0xD4EF3085
! 215: #define T44 0x04881D05
! 216: #define T45 0xD9D4D039
! 217: #define T46 0xE6DB99E5
! 218: #define T47 0x1FA27CF8
! 219: #define T48 0xC4AC5665
! 220: #define T49 0xF4292244
! 221: #define T50 0x432AFF97
! 222: #define T51 0xAB9423A7
! 223: #define T52 0xFC93A039
! 224: #define T53 0x655B59C3
! 225: #define T54 0x8F0CCC92
! 226: #define T55 0xFFEFF47D
! 227: #define T56 0x85845DD1
! 228: #define T57 0x6FA87E4F
! 229: #define T58 0xFE2CE6E0
! 230: #define T59 0xA3014314
! 231: #define T60 0x4E0811A1
! 232: #define T61 0xF7537E82
! 233: #define T62 0xBD3AF235
! 234: #define T63 0x2AD7D2BB
! 235: #define T64 0xEB86D391
! 236:
! 237: #define ROTL_SSE2(x, n) { \
! 238: __m128i s; \
! 239: s = _mm_srli_epi32(x, 32 - n); \
! 240: x = _mm_slli_epi32(x, n); \
! 241: x = _mm_or_si128(x, s); \
! 242: };
! 243:
! 244: #define ROTL_AVX2(x, n) { \
! 245: __m256i s; \
! 246: s = _mm256_srli_epi32(x, 32 - n); \
! 247: x = _mm256_slli_epi32(x, n); \
! 248: x = _mm256_or_si256(x, s); \
! 249: };
! 250:
! 251: #define F_SSE2(x, y, z) _mm_or_si128(_mm_and_si128(x, y), _mm_andnot_si128(x, z))
! 252: #define G_SSE2(x, y, z) _mm_or_si128(_mm_and_si128(x, z), _mm_andnot_si128(z, y))
! 253: #define H_SSE2(x, y, z) _mm_xor_si128(_mm_xor_si128(x, y), z)
! 254: #define I_SSE2(x, y, z) _mm_xor_si128(y, _mm_or_si128(x, _mm_andnot_si128(z, _mm_set1_epi32(0xffffffff))))
! 255:
! 256: #define F_AVX2(x, y, z) _mm256_or_si256(_mm256_and_si256(x, y), _mm256_andnot_si256(x, z))
! 257: #define G_AVX2(x, y, z) _mm256_or_si256(_mm256_and_si256(x, z), _mm256_andnot_si256(z, y))
! 258: #define H_AVX2(x, y, z) _mm256_xor_si256(_mm256_xor_si256(x, y), z)
! 259: #define I_AVX2(x, y, z) _mm256_xor_si256(y, _mm256_or_si256(x, _mm256_andnot_si256(z, _mm256_set1_epi32(0xffffffff))))
! 260:
! 261: #define SET_SSE2(step, a, b, c, d, x, s, ac) { \
! 262: a = _mm_add_epi32(_mm_add_epi32(a, _mm_add_epi32(x, _mm_set1_epi32(T##ac))), step##_SSE2(b, c, d)); \
! 263: ROTL_SSE2(a, s); \
! 264: a = _mm_add_epi32(a, b); \
! 265: }
! 266:
! 267: #define SET_AVX2(step, a, b, c, d, x, s, ac) { \
! 268: a = _mm256_add_epi32(_mm256_add_epi32(a, _mm256_add_epi32(x, _mm256_set1_epi32(T##ac))), step##_AVX2(b, c, d)); \
! 269: ROTL_AVX2(a, s); \
! 270: a = _mm256_add_epi32(a, b); \
! 271: }
! 272:
! 273: #define IA 0x67452301
! 274: #define IB 0xefcdab89
! 275: #define IC 0x98badcfe
! 276: #define ID 0x10325476
! 277:
! 278: #define GET_MD5_DATA(dest, src, pos) \
! 279: dest = \
! 280: ((uint32_t) src[pos + 0]) << 0 | \
! 281: ((uint32_t) src[pos + 1]) << 8 | \
! 282: ((uint32_t) src[pos + 2]) << 16 | \
! 283: ((uint32_t) src[pos + 3]) << 24
! 284:
! 285: #define GET_PMD5_DATA_SSE2(dest, src, pos) { \
! 286: uint32_t v0, v1, v2, v3; \
! 287: GET_MD5_DATA(v0, src[0], pos); \
! 288: GET_MD5_DATA(v1, src[1], pos); \
! 289: GET_MD5_DATA(v2, src[2], pos); \
! 290: GET_MD5_DATA(v3, src[3], pos); \
! 291: dest = _mm_setr_epi32(v0, v1, v2, v3); \
! 292: }
! 293:
! 294: #define GET_PMD5_DATA_AVX2(dest, src, pos) { \
! 295: uint32_t v0, v1, v2, v3; \
! 296: uint32_t v4, v5, v6, v7; \
! 297: GET_MD5_DATA(v0, src[0], pos); \
! 298: GET_MD5_DATA(v1, src[1], pos); \
! 299: GET_MD5_DATA(v2, src[2], pos); \
! 300: GET_MD5_DATA(v3, src[3], pos); \
! 301: GET_MD5_DATA(v4, src[4], pos); \
! 302: GET_MD5_DATA(v5, src[5], pos); \
! 303: GET_MD5_DATA(v6, src[6], pos); \
! 304: GET_MD5_DATA(v7, src[7], pos); \
! 305: dest = _mm256_setr_epi32(v0, v1, v2, v3, \
! 306: v4, v5, v6, v7); \
! 307: }
! 308:
! 309: #define PUT_MD5_DATA(dest, val, pos) { \
! 310: dest[pos + 0] = (val >> 0) & 0xff; \
! 311: dest[pos + 1] = (val >> 8) & 0xff; \
! 312: dest[pos + 2] = (val >> 16) & 0xff; \
! 313: dest[pos + 3] = (val >> 24) & 0xff; \
! 314: }
! 315:
! 316: const static uint8_t md5_padding[64] = {
! 317: 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 318: 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 319: 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 320: 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 321: 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 322: 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 323: 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 324: 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
! 325: };
! 326:
! 327: #ifdef PMD5_ALLOW_SSE2
! 328: __attribute__ ((target("sse2"))) MVSTATIC pmd5_status pmd5_init_all(pmd5_context * ctx)
! 329: {
! 330: int i;
! 331: for (i = 0; i < PMD5_SLOTS_MAX; i++) {
! 332: ctx->len[i] = 0;
! 333: }
! 334:
! 335: ctx->state_sse2[0] = _mm_set1_epi32(IA);
! 336: ctx->state_sse2[1] = _mm_set1_epi32(IB);
! 337: ctx->state_sse2[2] = _mm_set1_epi32(IC);
! 338: ctx->state_sse2[3] = _mm_set1_epi32(ID);
! 339:
! 340: return PMD5_SUCCESS;
! 341: }
! 342: #endif
! 343:
! 344: #ifdef PMD5_ALLOW_AVX2
! 345: __attribute__ ((target("avx2"))) MVSTATIC pmd5_status pmd5_init_all(pmd5_context * ctx)
! 346: {
! 347: int i;
! 348: for (i = 0; i < PMD5_SLOTS_MAX; i++) {
! 349: ctx->len[i] = 0;
! 350: }
! 351:
! 352: ctx->state_avx2[0] = _mm256_set1_epi32(IA);
! 353: ctx->state_avx2[1] = _mm256_set1_epi32(IB);
! 354: ctx->state_avx2[2] = _mm256_set1_epi32(IC);
! 355: ctx->state_avx2[3] = _mm256_set1_epi32(ID);
! 356:
! 357: return PMD5_SUCCESS;
! 358: }
! 359: #endif
! 360:
! 361: __attribute__ ((target("default"))) MVSTATIC pmd5_status pmd5_init_all(pmd5_context * ctx)
! 362: {
! 363: return PMD5_INVALID_SLOT;
! 364: }
! 365:
! 366: #ifdef PMD5_ALLOW_SSE2
! 367: __attribute__ ((target("sse2"))) MVSTATIC pmd5_status pmd5_set_slot(pmd5_context * ctx, int slot, uint32_t a, uint32_t b, uint32_t c, uint32_t d)
! 368: {
! 369: if ((slot >= PMD5_SLOTS_SSE2) || (slot < 0))
! 370: return PMD5_INVALID_SLOT;
! 371:
! 372: __attribute__ ((aligned(32))) uint32_t v[4][PMD5_SLOTS_SSE2];
! 373: int i;
! 374:
! 375: for (i = 0; i < 4; i++) {
! 376: _mm_store_si128((__m128i_u*)v[i], ctx->state_sse2[i]);
! 377: }
! 378:
! 379: v[0][slot] = a;
! 380: v[1][slot] = b;
! 381: v[2][slot] = c;
! 382: v[3][slot] = d;
! 383:
! 384: for (i = 0; i < 4; i++) {
! 385: ctx->state_sse2[i] = _mm_loadu_si128((__m128i_u*)v[i]);
! 386: }
! 387:
! 388: return PMD5_SUCCESS;
! 389: }
! 390: #endif
! 391:
! 392: #ifdef PMD5_ALLOW_AVX2
! 393: __attribute__ ((target("avx2"))) MVSTATIC pmd5_status pmd5_set_slot(pmd5_context * ctx, int slot, uint32_t a, uint32_t b, uint32_t c, uint32_t d)
! 394: {
! 395: if ((slot >= PMD5_SLOTS_AVX2) || (slot < 0))
! 396: return PMD5_INVALID_SLOT;
! 397:
! 398: __attribute__ ((aligned(32))) uint32_t v[4][PMD5_SLOTS_AVX2];
! 399: int i;
! 400:
! 401: for (i = 0; i < 4; i++) {
! 402: _mm256_store_si256((__m256i_u*)v[i], ctx->state_avx2[i]);
! 403: }
! 404:
! 405: v[0][slot] = a;
! 406: v[1][slot] = b;
! 407: v[2][slot] = c;
! 408: v[3][slot] = d;
! 409:
! 410: for (i = 0; i < 4; i++) {
! 411: ctx->state_avx2[i] = _mm256_lddqu_si256((__m256i_u*)v[i]);
! 412: }
! 413:
! 414: return PMD5_SUCCESS;
! 415: }
! 416: #endif
! 417:
! 418: __attribute__ ((target("default"))) MVSTATIC pmd5_status pmd5_set_slot(pmd5_context * ctx, int slot, uint32_t a, uint32_t b, uint32_t c, uint32_t d)
! 419: {
! 420: return PMD5_INVALID_SLOT;
! 421: }
! 422:
! 423: #ifdef PMD5_ALLOW_SSE2
! 424: __attribute__ ((target("sse2"))) MVSTATIC pmd5_status pmd5_get_slot(const pmd5_context * ctx, int slot, uint32_t* a, uint32_t* b, uint32_t* c, uint32_t* d)
! 425: {
! 426: if ((slot >= PMD5_SLOTS_SSE2) || (slot < 0))
! 427: return PMD5_INVALID_SLOT;
! 428:
! 429: __attribute__ ((aligned(32))) uint32_t v[4][PMD5_SLOTS_SSE2];
! 430: int i;
! 431:
! 432: for (i = 0; i < 4; i++) {
! 433: _mm_store_si128((__m128i_u*)v[i], ctx->state_sse2[i]);
! 434: }
! 435:
! 436: *a = v[0][slot];
! 437: *b = v[1][slot];
! 438: *c = v[2][slot];
! 439: *d = v[3][slot];
! 440:
! 441: return PMD5_SUCCESS;
! 442: }
! 443: #endif
! 444:
! 445: #ifdef PMD5_ALLOW_AVX2
! 446: __attribute__ ((target("avx2"))) MVSTATIC pmd5_status pmd5_get_slot(const pmd5_context * ctx, int slot, uint32_t* a, uint32_t* b, uint32_t* c, uint32_t* d)
! 447: {
! 448: if ((slot >= PMD5_SLOTS_AVX2) || (slot < 0))
! 449: return PMD5_INVALID_SLOT;
! 450:
! 451: __attribute__ ((aligned(32))) uint32_t v[4][PMD5_SLOTS_AVX2];
! 452: int i;
! 453:
! 454: for (i = 0; i < 4; i++) {
! 455: _mm256_store_si256((__m256i_u*)v[i], ctx->state_avx2[i]);
! 456: }
! 457:
! 458: *a = v[0][slot];
! 459: *b = v[1][slot];
! 460: *c = v[2][slot];
! 461: *d = v[3][slot];
! 462:
! 463: return PMD5_SUCCESS;
! 464: }
! 465: #endif
! 466:
! 467: __attribute__ ((target("default"))) MVSTATIC pmd5_status pmd5_get_slot(const pmd5_context * ctx, int slot, uint32_t* a, uint32_t* b, uint32_t* c, uint32_t* d)
! 468: {
! 469: return PMD5_INVALID_SLOT;
! 470: }
! 471:
! 472: static pmd5_status pmd5_init_slot(pmd5_context * ctx, int slot)
! 473: {
! 474: return pmd5_set_slot(ctx, slot, IA, IB, IC, ID);
! 475: }
! 476:
! 477: #ifdef PMD5_ALLOW_SSE2
! 478: __attribute__ ((target("sse2"))) MVSTATIC void pmd5_process(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX])
! 479: {
! 480: __m128i W[MD5_DIGEST_LEN], a, b, c, d;
! 481:
! 482: GET_PMD5_DATA_SSE2(W[ 0], data, 0);
! 483: GET_PMD5_DATA_SSE2(W[ 1], data, 4);
! 484: GET_PMD5_DATA_SSE2(W[ 2], data, 8);
! 485: GET_PMD5_DATA_SSE2(W[ 3], data, 12);
! 486: GET_PMD5_DATA_SSE2(W[ 4], data, 16);
! 487: GET_PMD5_DATA_SSE2(W[ 5], data, 20);
! 488: GET_PMD5_DATA_SSE2(W[ 6], data, 24);
! 489: GET_PMD5_DATA_SSE2(W[ 7], data, 28);
! 490: GET_PMD5_DATA_SSE2(W[ 8], data, 32);
! 491: GET_PMD5_DATA_SSE2(W[ 9], data, 36);
! 492: GET_PMD5_DATA_SSE2(W[10], data, 40);
! 493: GET_PMD5_DATA_SSE2(W[11], data, 44);
! 494: GET_PMD5_DATA_SSE2(W[12], data, 48);
! 495: GET_PMD5_DATA_SSE2(W[13], data, 52);
! 496: GET_PMD5_DATA_SSE2(W[14], data, 56);
! 497: GET_PMD5_DATA_SSE2(W[15], data, 60);
! 498:
! 499: a = ctx->state_sse2[0];
! 500: b = ctx->state_sse2[1];
! 501: c = ctx->state_sse2[2];
! 502: d = ctx->state_sse2[3];
! 503:
! 504: SET_SSE2(F, a, b, c, d, W[ 0], S11, 1);
! 505: SET_SSE2(F, d, a, b, c, W[ 1], S12, 2);
! 506: SET_SSE2(F, c, d, a, b, W[ 2], S13, 3);
! 507: SET_SSE2(F, b, c, d, a, W[ 3], S14, 4);
! 508: SET_SSE2(F, a, b, c, d, W[ 4], S11, 5);
! 509: SET_SSE2(F, d, a, b, c, W[ 5], S12, 6);
! 510: SET_SSE2(F, c, d, a, b, W[ 6], S13, 7);
! 511: SET_SSE2(F, b, c, d, a, W[ 7], S14, 8);
! 512: SET_SSE2(F, a, b, c, d, W[ 8], S11, 9);
! 513: SET_SSE2(F, d, a, b, c, W[ 9], S12, 10);
! 514: SET_SSE2(F, c, d, a, b, W[10], S13, 11);
! 515: SET_SSE2(F, b, c, d, a, W[11], S14, 12);
! 516: SET_SSE2(F, a, b, c, d, W[12], S11, 13);
! 517: SET_SSE2(F, d, a, b, c, W[13], S12, 14);
! 518: SET_SSE2(F, c, d, a, b, W[14], S13, 15);
! 519: SET_SSE2(F, b, c, d, a, W[15], S14, 16);
! 520:
! 521: SET_SSE2(G, a, b, c, d, W[ 1], S21, 17);
! 522: SET_SSE2(G, d, a, b, c, W[ 6], S22, 18);
! 523: SET_SSE2(G, c, d, a, b, W[11], S23, 19);
! 524: SET_SSE2(G, b, c, d, a, W[ 0], S24, 20);
! 525: SET_SSE2(G, a, b, c, d, W[ 5], S21, 21);
! 526: SET_SSE2(G, d, a, b, c, W[10], S22, 22);
! 527: SET_SSE2(G, c, d, a, b, W[15], S23, 23);
! 528: SET_SSE2(G, b, c, d, a, W[ 4], S24, 24);
! 529: SET_SSE2(G, a, b, c, d, W[ 9], S21, 25);
! 530: SET_SSE2(G, d, a, b, c, W[14], S22, 26);
! 531: SET_SSE2(G, c, d, a, b, W[ 3], S23, 27);
! 532: SET_SSE2(G, b, c, d, a, W[ 8], S24, 28);
! 533: SET_SSE2(G, a, b, c, d, W[13], S21, 29);
! 534: SET_SSE2(G, d, a, b, c, W[ 2], S22, 30);
! 535: SET_SSE2(G, c, d, a, b, W[ 7], S23, 31);
! 536: SET_SSE2(G, b, c, d, a, W[12], S24, 32);
! 537:
! 538: SET_SSE2(H, a, b, c, d, W[ 5], S31, 33);
! 539: SET_SSE2(H, d, a, b, c, W[ 8], S32, 34);
! 540: SET_SSE2(H, c, d, a, b, W[11], S33, 35);
! 541: SET_SSE2(H, b, c, d, a, W[14], S34, 36);
! 542: SET_SSE2(H, a, b, c, d, W[ 1], S31, 37);
! 543: SET_SSE2(H, d, a, b, c, W[ 4], S32, 38);
! 544: SET_SSE2(H, c, d, a, b, W[ 7], S33, 39);
! 545: SET_SSE2(H, b, c, d, a, W[10], S34, 40);
! 546: SET_SSE2(H, a, b, c, d, W[13], S31, 41);
! 547: SET_SSE2(H, d, a, b, c, W[ 0], S32, 42);
! 548: SET_SSE2(H, c, d, a, b, W[ 3], S33, 43);
! 549: SET_SSE2(H, b, c, d, a, W[ 6], S34, 44);
! 550: SET_SSE2(H, a, b, c, d, W[ 9], S31, 45);
! 551: SET_SSE2(H, d, a, b, c, W[12], S32, 46);
! 552: SET_SSE2(H, c, d, a, b, W[15], S33, 47);
! 553: SET_SSE2(H, b, c, d, a, W[ 2], S34, 48);
! 554:
! 555: SET_SSE2(I, a, b, c, d, W[ 0], S41, 49);
! 556: SET_SSE2(I, d, a, b, c, W[ 7], S42, 50);
! 557: SET_SSE2(I, c, d, a, b, W[14], S43, 51);
! 558: SET_SSE2(I, b, c, d, a, W[ 5], S44, 52);
! 559: SET_SSE2(I, a, b, c, d, W[12], S41, 53);
! 560: SET_SSE2(I, d, a, b, c, W[ 3], S42, 54);
! 561: SET_SSE2(I, c, d, a, b, W[10], S43, 55);
! 562: SET_SSE2(I, b, c, d, a, W[ 1], S44, 56);
! 563: SET_SSE2(I, a, b, c, d, W[ 8], S41, 57);
! 564: SET_SSE2(I, d, a, b, c, W[15], S42, 58);
! 565: SET_SSE2(I, c, d, a, b, W[ 6], S43, 59);
! 566: SET_SSE2(I, b, c, d, a, W[13], S44, 60);
! 567: SET_SSE2(I, a, b, c, d, W[ 4], S41, 61);
! 568: SET_SSE2(I, d, a, b, c, W[11], S42, 62);
! 569: SET_SSE2(I, c, d, a, b, W[ 2], S43, 63);
! 570: SET_SSE2(I, b, c, d, a, W[ 9], S44, 64);
! 571:
! 572: ctx->state_sse2[0] = _mm_add_epi32(ctx->state_sse2[0], a);
! 573: ctx->state_sse2[1] = _mm_add_epi32(ctx->state_sse2[1], b);
! 574: ctx->state_sse2[2] = _mm_add_epi32(ctx->state_sse2[2], c);
! 575: ctx->state_sse2[3] = _mm_add_epi32(ctx->state_sse2[3], d);
! 576: }
! 577: #endif
! 578:
! 579: #ifdef PMD5_ALLOW_AVX2
! 580: __attribute__ ((target("avx2"))) MVSTATIC void pmd5_process(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX])
! 581: {
! 582: __m256i W[MD5_DIGEST_LEN], a, b, c, d;
! 583:
! 584: GET_PMD5_DATA_AVX2(W[ 0], data, 0);
! 585: GET_PMD5_DATA_AVX2(W[ 1], data, 4);
! 586: GET_PMD5_DATA_AVX2(W[ 2], data, 8);
! 587: GET_PMD5_DATA_AVX2(W[ 3], data, 12);
! 588: GET_PMD5_DATA_AVX2(W[ 4], data, 16);
! 589: GET_PMD5_DATA_AVX2(W[ 5], data, 20);
! 590: GET_PMD5_DATA_AVX2(W[ 6], data, 24);
! 591: GET_PMD5_DATA_AVX2(W[ 7], data, 28);
! 592: GET_PMD5_DATA_AVX2(W[ 8], data, 32);
! 593: GET_PMD5_DATA_AVX2(W[ 9], data, 36);
! 594: GET_PMD5_DATA_AVX2(W[10], data, 40);
! 595: GET_PMD5_DATA_AVX2(W[11], data, 44);
! 596: GET_PMD5_DATA_AVX2(W[12], data, 48);
! 597: GET_PMD5_DATA_AVX2(W[13], data, 52);
! 598: GET_PMD5_DATA_AVX2(W[14], data, 56);
! 599: GET_PMD5_DATA_AVX2(W[15], data, 60);
! 600:
! 601: a = ctx->state_avx2[0];
! 602: b = ctx->state_avx2[1];
! 603: c = ctx->state_avx2[2];
! 604: d = ctx->state_avx2[3];
! 605:
! 606: SET_AVX2(F, a, b, c, d, W[ 0], S11, 1);
! 607: SET_AVX2(F, d, a, b, c, W[ 1], S12, 2);
! 608: SET_AVX2(F, c, d, a, b, W[ 2], S13, 3);
! 609: SET_AVX2(F, b, c, d, a, W[ 3], S14, 4);
! 610: SET_AVX2(F, a, b, c, d, W[ 4], S11, 5);
! 611: SET_AVX2(F, d, a, b, c, W[ 5], S12, 6);
! 612: SET_AVX2(F, c, d, a, b, W[ 6], S13, 7);
! 613: SET_AVX2(F, b, c, d, a, W[ 7], S14, 8);
! 614: SET_AVX2(F, a, b, c, d, W[ 8], S11, 9);
! 615: SET_AVX2(F, d, a, b, c, W[ 9], S12, 10);
! 616: SET_AVX2(F, c, d, a, b, W[10], S13, 11);
! 617: SET_AVX2(F, b, c, d, a, W[11], S14, 12);
! 618: SET_AVX2(F, a, b, c, d, W[12], S11, 13);
! 619: SET_AVX2(F, d, a, b, c, W[13], S12, 14);
! 620: SET_AVX2(F, c, d, a, b, W[14], S13, 15);
! 621: SET_AVX2(F, b, c, d, a, W[15], S14, 16);
! 622:
! 623: SET_AVX2(G, a, b, c, d, W[ 1], S21, 17);
! 624: SET_AVX2(G, d, a, b, c, W[ 6], S22, 18);
! 625: SET_AVX2(G, c, d, a, b, W[11], S23, 19);
! 626: SET_AVX2(G, b, c, d, a, W[ 0], S24, 20);
! 627: SET_AVX2(G, a, b, c, d, W[ 5], S21, 21);
! 628: SET_AVX2(G, d, a, b, c, W[10], S22, 22);
! 629: SET_AVX2(G, c, d, a, b, W[15], S23, 23);
! 630: SET_AVX2(G, b, c, d, a, W[ 4], S24, 24);
! 631: SET_AVX2(G, a, b, c, d, W[ 9], S21, 25);
! 632: SET_AVX2(G, d, a, b, c, W[14], S22, 26);
! 633: SET_AVX2(G, c, d, a, b, W[ 3], S23, 27);
! 634: SET_AVX2(G, b, c, d, a, W[ 8], S24, 28);
! 635: SET_AVX2(G, a, b, c, d, W[13], S21, 29);
! 636: SET_AVX2(G, d, a, b, c, W[ 2], S22, 30);
! 637: SET_AVX2(G, c, d, a, b, W[ 7], S23, 31);
! 638: SET_AVX2(G, b, c, d, a, W[12], S24, 32);
! 639:
! 640: SET_AVX2(H, a, b, c, d, W[ 5], S31, 33);
! 641: SET_AVX2(H, d, a, b, c, W[ 8], S32, 34);
! 642: SET_AVX2(H, c, d, a, b, W[11], S33, 35);
! 643: SET_AVX2(H, b, c, d, a, W[14], S34, 36);
! 644: SET_AVX2(H, a, b, c, d, W[ 1], S31, 37);
! 645: SET_AVX2(H, d, a, b, c, W[ 4], S32, 38);
! 646: SET_AVX2(H, c, d, a, b, W[ 7], S33, 39);
! 647: SET_AVX2(H, b, c, d, a, W[10], S34, 40);
! 648: SET_AVX2(H, a, b, c, d, W[13], S31, 41);
! 649: SET_AVX2(H, d, a, b, c, W[ 0], S32, 42);
! 650: SET_AVX2(H, c, d, a, b, W[ 3], S33, 43);
! 651: SET_AVX2(H, b, c, d, a, W[ 6], S34, 44);
! 652: SET_AVX2(H, a, b, c, d, W[ 9], S31, 45);
! 653: SET_AVX2(H, d, a, b, c, W[12], S32, 46);
! 654: SET_AVX2(H, c, d, a, b, W[15], S33, 47);
! 655: SET_AVX2(H, b, c, d, a, W[ 2], S34, 48);
! 656:
! 657: SET_AVX2(I, a, b, c, d, W[ 0], S41, 49);
! 658: SET_AVX2(I, d, a, b, c, W[ 7], S42, 50);
! 659: SET_AVX2(I, c, d, a, b, W[14], S43, 51);
! 660: SET_AVX2(I, b, c, d, a, W[ 5], S44, 52);
! 661: SET_AVX2(I, a, b, c, d, W[12], S41, 53);
! 662: SET_AVX2(I, d, a, b, c, W[ 3], S42, 54);
! 663: SET_AVX2(I, c, d, a, b, W[10], S43, 55);
! 664: SET_AVX2(I, b, c, d, a, W[ 1], S44, 56);
! 665: SET_AVX2(I, a, b, c, d, W[ 8], S41, 57);
! 666: SET_AVX2(I, d, a, b, c, W[15], S42, 58);
! 667: SET_AVX2(I, c, d, a, b, W[ 6], S43, 59);
! 668: SET_AVX2(I, b, c, d, a, W[13], S44, 60);
! 669: SET_AVX2(I, a, b, c, d, W[ 4], S41, 61);
! 670: SET_AVX2(I, d, a, b, c, W[11], S42, 62);
! 671: SET_AVX2(I, c, d, a, b, W[ 2], S43, 63);
! 672: SET_AVX2(I, b, c, d, a, W[ 9], S44, 64);
! 673:
! 674: ctx->state_avx2[0] = _mm256_add_epi32(ctx->state_avx2[0], a);
! 675: ctx->state_avx2[1] = _mm256_add_epi32(ctx->state_avx2[1], b);
! 676: ctx->state_avx2[2] = _mm256_add_epi32(ctx->state_avx2[2], c);
! 677: ctx->state_avx2[3] = _mm256_add_epi32(ctx->state_avx2[3], d);
! 678: }
! 679: #endif
! 680:
! 681: __attribute__ ((target("default"))) MVSTATIC void pmd5_process(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX])
! 682: {
! 683: }
! 684:
! 685: static pmd5_status pmd5_update_all_simple(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX], uint64_t length, uint64_t stride)
! 686: {
! 687: const uint8_t * ptrs[PMD5_SLOTS_MAX];
! 688:
! 689: if (!length) return PMD5_SUCCESS;
! 690:
! 691: int slots = pmd5_slots();
! 692:
! 693: if (!stride) stride = 64;
! 694:
! 695: int i;
! 696: for (i = 0; i < slots; i++) {
! 697: ptrs[i] = data[i];
! 698: ctx->len[i] += length;
! 699: if (!ptrs[i]) ptrs[i] = md5_padding;
! 700: }
! 701:
! 702: while (length >= 64) {
! 703: pmd5_process(ctx, ptrs);
! 704: length -= 64;
! 705: for (i = 0; i < slots; i++) {
! 706: if (data[i]) ptrs[i] += stride;
! 707: }
! 708: }
! 709:
! 710: if (length) return PMD5_UNALIGNED_UPDATE;
! 711:
! 712: for (i = 0; i < slots; i++) {
! 713: if (data[i]) data[i] = ptrs[i];
! 714: }
! 715:
! 716: return PMD5_SUCCESS;
! 717: }
! 718:
! 719: static pmd5_status pmd5_update_all(pmd5_context * ctx, const uint8_t * data[PMD5_SLOTS_MAX], uint64_t lengths[PMD5_SLOTS_MAX])
! 720: {
! 721: uint64_t length = 0;
! 722: int slots = pmd5_slots();
! 723:
! 724: int i;
! 725: for (i = 0; i < slots; i++) {
! 726: if ((length == 0) || (lengths[i] < length)) length = lengths[i];
! 727: }
! 728:
! 729: for (i = 0; i < slots; i++) {
! 730: lengths[i] -= length;
! 731: }
! 732:
! 733: return pmd5_update_all_simple(ctx, data, length, 0);
! 734: }
! 735:
! 736: static pmd5_status pmd5_finish_slot_with_extra(pmd5_context * pctx, uint8_t digest[MD5_DIGEST_LEN], int slot, const uint8_t * data, uint64_t length)
! 737: {
! 738: MD5_CTX ctx;
! 739:
! 740: if ((slot >= pmd5_slots()) || (slot < 0))
! 741: return PMD5_INVALID_SLOT;
! 742:
! 743: pmd5_to_md5(pctx, &ctx, slot);
! 744: if (data && length) {
! 745: MD5_Update(&ctx, data, length);
! 746: }
! 747: MD5_Final(digest, &ctx);
! 748:
! 749: return PMD5_SUCCESS;
! 750: }
! 751:
! 752: static pmd5_status pmd5_finish_slot(pmd5_context * pctx, uint8_t digest[MD5_DIGEST_LEN], int slot)
! 753: {
! 754: return pmd5_finish_slot_with_extra(pctx, digest, slot, NULL, 0);
! 755: }
! 756:
! 757: static pmd5_status pmd5_finish_all(pmd5_context * ctx, uint8_t digests[PMD5_SLOTS_MAX][MD5_DIGEST_LEN])
! 758: {
! 759: int i;
! 760: for (i = 0; i < pmd5_slots(); i++) {
! 761: pmd5_finish_slot_with_extra(ctx, digests[i], i, NULL, 0);
! 762: }
! 763: return PMD5_SUCCESS;
! 764: }
! 765:
! 766: static pmd5_status md5_to_pmd5(const MD5_CTX * ctx, pmd5_context * pctx, int slot)
! 767: {
! 768: if ((slot >= pmd5_slots()) || (slot < 0))
! 769: return PMD5_INVALID_SLOT;
! 770:
! 771: // TODO This function ignores buffered but as of yet unhashed data. We're not using this function, just noting.
! 772:
! 773: #ifdef USE_OPENSSL
! 774: pctx->len[slot] = (ctx->Nl >> 3) + ((uint64_t)ctx->Nh << 29);
! 775: #else
! 776: pctx->len[slot] = ctx->totalN + ((uint64_t)ctx->totalN2 << 32);
! 777: #endif
! 778: return pmd5_set_slot(pctx, slot, (uint32_t)ctx->A, (uint32_t)ctx->B, (uint32_t)ctx->C, (uint32_t)ctx->D);
! 779: }
! 780:
! 781: static pmd5_status pmd5_to_md5(const pmd5_context * pctx, MD5_CTX * ctx, int slot)
! 782: {
! 783: if ((slot >= pmd5_slots()) || (slot < 0))
! 784: return PMD5_INVALID_SLOT;
! 785:
! 786: MD5_Init(ctx);
! 787:
! 788: #ifdef USE_OPENSSL
! 789: ctx->Nl = (pctx->len[slot] << 3) & 0xFFFFFFFF;
! 790: ctx->Nh = pctx->len[slot] >> 29;
! 791:
! 792: uint32_t a, b, c, d;
! 793: pmd5_status ret = pmd5_get_slot(pctx, slot, &a, &b, &c, &d);
! 794: if (ret == PMD5_SUCCESS) {
! 795: ctx->A = a;
! 796: ctx->B = b;
! 797: ctx->C = c;
! 798: ctx->D = d;
! 799: }
! 800: return ret;
! 801: #else
! 802: ctx->totalN = pctx->len[slot] & 0xFFFFFFFF;
! 803: ctx->totalN2 = pctx->len[slot] >> 32;
! 804: return pmd5_get_slot(pctx, slot, &ctx->A, &ctx->B, &ctx->C, &ctx->D);
! 805: #endif
! 806: }
! 807:
! 808: /* With GCC 10 putting these implementations inside 'extern "C"' causes an
! 809: assembler error. That worked fine on GCC 5-9 and clang 6-10...
! 810: */
! 811:
! 812: static inline int md5_parallel_slots_cpp()
! 813: {
! 814: int slots = pmd5_slots();
! 815: if (slots == 0) return 1;
! 816: return slots;
! 817: }
! 818:
! 819: static inline int md5_parallel_cpp(int streams, char** buf, int* len, char** sum, char* pre4, char* post4)
! 820: {
! 821: int slots = md5_parallel_slots_cpp();
! 822: if ((streams < 1) || (streams > slots)) return 0;
! 823: if (pre4 && post4) return 0;
! 824:
! 825: if (slots == 1) {
! 826: MD5_CTX ctx;
! 827: MD5_Init(&ctx);
! 828: if (pre4) {
! 829: MD5_Update(&ctx, (const unsigned char*)pre4, 4);
! 830: }
! 831: MD5_Update(&ctx, (const unsigned char*)buf[0], len[0]);
! 832: if (post4) {
! 833: MD5_Update(&ctx, (const unsigned char*)post4, 4);
! 834: }
! 835: if (sum[0]) {
! 836: MD5_Final((uint8_t*)sum[0], &ctx);
! 837: }
! 838: return 0;
! 839: }
! 840:
! 841: int i;
! 842: int active[PMD5_SLOTS_MAX];
! 843: char* buffers[PMD5_SLOTS_MAX];
! 844: uint64_t left[PMD5_SLOTS_MAX];
! 845: for (i = 0; i < PMD5_SLOTS_MAX; i++) {
! 846: active[i] = streams > i;
! 847: if (i < streams) {
! 848: buffers[i] = buf[i];
! 849: left[i] = (uint64_t)len[i];
! 850: } else {
! 851: buffers[i] = NULL;
! 852: left[i] = 0;
! 853: }
! 854: }
! 855: MD5_CTX results[PMD5_SLOTS_MAX];
! 856:
! 857: pmd5_context ctx_simd;
! 858: if (pmd5_init_all(&ctx_simd) != PMD5_SUCCESS) return 0;
! 859:
! 860: if (pre4) {
! 861: char temp_buffers[PMD5_SLOTS_MAX][64];
! 862: int have_any = 0;
! 863: for (i = 0; i < slots; i++) {
! 864: if (active[i]) {
! 865: if (left[i] < 60) {
! 866: MD5_Init(&results[i]);
! 867: MD5_Update(&results[i], (const unsigned char*)pre4, 4);
! 868: MD5_Update(&results[i], (const unsigned char*)buf[i], left[i]);
! 869: active[i] = 0;
! 870: left[i] = 0;
! 871: } else {
! 872: memcpy(temp_buffers[i], pre4, 4);
! 873: memcpy(temp_buffers[i] + 4, buffers[i], 60);
! 874: buffers[i] += 60;
! 875: left[i] -= 60;
! 876: have_any = 1;
! 877: }
! 878: }
! 879: }
! 880:
! 881: if (have_any) {
! 882: char* ptrs[PMD5_SLOTS_MAX];
! 883: for (i = 0; i < PMD5_SLOTS_MAX; i++) {
! 884: ptrs[i] = &temp_buffers[i][0];
! 885: }
! 886: if (pmd5_update_all_simple(&ctx_simd, (const uint8_t**)ptrs, 64, 0) != PMD5_SUCCESS) {
! 887: return 0;
! 888: }
! 889: }
! 890: }
! 891:
! 892: int failed = 0;
! 893: while (true) {
! 894: for (i = 0; i < slots; i++) {
! 895: if (active[i] && (left[i] < 64)) {
! 896: if (pmd5_to_md5(&ctx_simd, &results[i], i) != PMD5_SUCCESS) {
! 897: failed = 1;
! 898: }
! 899: active[i] = 0;
! 900: }
! 901: }
! 902:
! 903: uint64_t shortest = 0;
! 904: for (i = 0; i < slots; i++) {
! 905: if (!active[i]) {
! 906: buffers[i] = NULL;
! 907: } else if ((shortest == 0) || (left[i] < shortest)) {
! 908: shortest = left[i];
! 909: }
! 910: }
! 911:
! 912: if (shortest > 0) {
! 913: shortest = shortest & ~63;
! 914: if (pmd5_update_all_simple(&ctx_simd, (const uint8_t**)buffers, shortest, 0) != PMD5_SUCCESS) {
! 915: failed = 1;
! 916: }
! 917: for (i = 0; i < slots; i++) {
! 918: if (active[i]) {
! 919: left[i] -= shortest;
! 920: }
! 921: }
! 922: }
! 923:
! 924: if (failed) {
! 925: return 0;
! 926: } else {
! 927: int have_any = 0;
! 928: for (i = 0; i < slots; i++) {
! 929: have_any |= active[i];
! 930: }
! 931: if (!have_any) {
! 932: break;
! 933: }
! 934: }
! 935: }
! 936:
! 937: for (i = 0; i < slots; i++) {
! 938: if (i < streams) {
! 939: if (left[i] > 0) {
! 940: // buffer[i] == NULL here
! 941: MD5_Update(&results[i], (const unsigned char*)buf[i] + len[i] - left[i], left[i]);
! 942: }
! 943: if (post4) {
! 944: MD5_Update(&results[i], (const unsigned char*)post4, 4);
! 945: }
! 946: if (sum[i]) {
! 947: MD5_Final((uint8_t*)sum[i], &results[i]);
! 948: }
! 949: }
! 950: }
! 951:
! 952: return 1;
! 953: }
! 954:
! 955: // each pmd5_context needs to be 32-byte aligned
! 956: #define MD5P8_Contexts_simd(ctx, index) ((pmd5_context*)((((uintptr_t)((ctx)->context_storage) + 31) & ~31) + (index)*((sizeof(pmd5_context) + 31) & ~31)))
! 957:
! 958: static inline void MD5P8_Init_cpp(MD5P8_CTX *ctx)
! 959: {
! 960: int i;
! 961: for (i = 0; i < (pmd5_slots() == PMD5_SLOTS_AVX2 ? 1 : 2); i++) {
! 962: pmd5_init_all(MD5P8_Contexts_simd(ctx, i));
! 963: }
! 964: ctx->used = 0;
! 965: ctx->next = 0;
! 966: }
! 967:
! 968: static inline void MD5P8_Update_cpp(MD5P8_CTX *ctx, const uchar *input, uint32 length)
! 969: {
! 970: int slots = pmd5_slots();
! 971: uint32 pos = 0;
! 972:
! 973: if ((ctx->used) || (length < 512)) {
! 974: int cpy = MIN(length, 512 - ctx->used);
! 975: memcpy(&ctx->buffer[ctx->used], input, cpy);
! 976: ctx->used += cpy;
! 977: length -= cpy;
! 978: pos += cpy;
! 979:
! 980: if (ctx->used == 512) {
! 981: if (slots == PMD5_SLOTS_AVX2) {
! 982: const uint8_t* ptrs[PMD5_SLOTS_MAX] = {
! 983: (uint8_t*)ctx->buffer,
! 984: (uint8_t*)(ctx->buffer + 64),
! 985: (uint8_t*)(ctx->buffer + 128),
! 986: (uint8_t*)(ctx->buffer + 192),
! 987: (uint8_t*)(ctx->buffer + 256),
! 988: (uint8_t*)(ctx->buffer + 320),
! 989: (uint8_t*)(ctx->buffer + 384),
! 990: (uint8_t*)(ctx->buffer + 448)
! 991: };
! 992: pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 0), ptrs, 64, 0);
! 993: } else {
! 994: const uint8_t* ptrs1[PMD5_SLOTS_MAX] = {
! 995: (uint8_t*)ctx->buffer,
! 996: (uint8_t*)(ctx->buffer + 64),
! 997: (uint8_t*)(ctx->buffer + 128),
! 998: (uint8_t*)(ctx->buffer + 192)
! 999: };
! 1000: const uint8_t* ptrs2[PMD5_SLOTS_MAX] = {
! 1001: (uint8_t*)(ctx->buffer + 256),
! 1002: (uint8_t*)(ctx->buffer + 320),
! 1003: (uint8_t*)(ctx->buffer + 384),
! 1004: (uint8_t*)(ctx->buffer + 448)
! 1005: };
! 1006: pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 0), ptrs1, 64, 0);
! 1007: pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 1), ptrs2, 64, 0);
! 1008: }
! 1009: ctx->used = 0;
! 1010: }
! 1011: }
! 1012:
! 1013: if (length >= 512) {
! 1014: uint32 blocks = length / 512;
! 1015: if (slots == PMD5_SLOTS_AVX2) {
! 1016: const uint8_t* ptrs[8] = {
! 1017: (uint8_t*)(input + pos),
! 1018: (uint8_t*)(input + pos + 64),
! 1019: (uint8_t*)(input + pos + 128),
! 1020: (uint8_t*)(input + pos + 192),
! 1021: (uint8_t*)(input + pos + 256),
! 1022: (uint8_t*)(input + pos + 320),
! 1023: (uint8_t*)(input + pos + 384),
! 1024: (uint8_t*)(input + pos + 448)
! 1025: };
! 1026: pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 0), ptrs, blocks * 64, 512);
! 1027: } else {
! 1028: const uint8_t* ptrs1[4] = {
! 1029: (uint8_t*)(input + pos),
! 1030: (uint8_t*)(input + pos + 64),
! 1031: (uint8_t*)(input + pos + 128),
! 1032: (uint8_t*)(input + pos + 192)
! 1033: };
! 1034: const uint8_t* ptrs2[4] = {
! 1035: (uint8_t*)(input + pos + 256),
! 1036: (uint8_t*)(input + pos + 320),
! 1037: (uint8_t*)(input + pos + 384),
! 1038: (uint8_t*)(input + pos + 448)
! 1039: };
! 1040: pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 0), ptrs1, blocks * 64, 512);
! 1041: pmd5_update_all_simple(MD5P8_Contexts_simd(ctx, 1), ptrs2, blocks * 64, 512);
! 1042: }
! 1043: pos += blocks * 512;
! 1044: length -= blocks * 512;
! 1045: }
! 1046:
! 1047: if (length) {
! 1048: memcpy(ctx->buffer, &input[pos], length);
! 1049: ctx->used = length;
! 1050: }
! 1051: }
! 1052:
! 1053: static inline void MD5P8_Final_cpp(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx)
! 1054: {
! 1055: int i;
! 1056: uint32 low = 0, high = 0, sub = ctx->used ? 512 - ctx->used : 0;
! 1057: if (ctx->used) {
! 1058: uchar tmp[512];
! 1059: memset(tmp, 0, 512);
! 1060: MD5P8_Update(ctx, tmp, 512 - ctx->used);
! 1061: }
! 1062:
! 1063: uchar state[34*4] = {0};
! 1064:
! 1065: MD5_CTX tmp;
! 1066: for (i = 0; i < 8; i++) {
! 1067: if (pmd5_slots() == PMD5_SLOTS_AVX2) {
! 1068: pmd5_to_md5(MD5P8_Contexts_simd(ctx, 0), &tmp, i);
! 1069: } else if (i < 4) {
! 1070: pmd5_to_md5(MD5P8_Contexts_simd(ctx, 0), &tmp, i);
! 1071: } else {
! 1072: pmd5_to_md5(MD5P8_Contexts_simd(ctx, 1), &tmp, i - 4);
! 1073: }
! 1074: #ifdef USE_OPENSSL
! 1075: if (low + tmp.Nl < low) high++;
! 1076: low += tmp.Nl;
! 1077: high += tmp.Nh;
! 1078: #else
! 1079: if (low + tmp.totalN < low) high++;
! 1080: low += tmp.totalN;
! 1081: high += tmp.totalN2;
! 1082: #endif
! 1083: SIVALu(state, i*16, tmp.A);
! 1084: SIVALu(state, i*16 + 4, tmp.B);
! 1085: SIVALu(state, i*16 + 8, tmp.C);
! 1086: SIVALu(state, i*16 + 12, tmp.D);
! 1087: }
! 1088:
! 1089: #ifndef USE_OPENSSL
! 1090: high = (low >> 29) | (high << 3);
! 1091: low = (low << 3);
! 1092: #endif
! 1093:
! 1094: sub <<= 3;
! 1095: if (low - sub > low) high--;
! 1096: low -= sub;
! 1097:
! 1098: SIVALu(state, 32*4, low);
! 1099: SIVALu(state, 33*4, high);
! 1100:
! 1101: MD5_CTX md;
! 1102: MD5_Init(&md);
! 1103: MD5_Update(&md, state, 34*4);
! 1104: MD5_Final(digest, &md);
! 1105: }
! 1106:
! 1107: extern "C" {
! 1108:
! 1109: int md5_parallel_slots()
! 1110: {
! 1111: return md5_parallel_slots_cpp();
! 1112: }
! 1113:
! 1114: int md5_parallel(int streams, char** buf, int* len, char** sum, char* pre4, char* post4)
! 1115: {
! 1116: return md5_parallel_cpp(streams, buf, len, sum, pre4, post4);
! 1117: }
! 1118:
! 1119: void MD5P8_Init(MD5P8_CTX *ctx)
! 1120: {
! 1121: MD5P8_Init_cpp(ctx);
! 1122: }
! 1123:
! 1124: void MD5P8_Update(MD5P8_CTX *ctx, const uchar *input, uint32 length)
! 1125: {
! 1126: MD5P8_Update_cpp(ctx, input, length);
! 1127: }
! 1128:
! 1129: void MD5P8_Final(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx)
! 1130: {
! 1131: MD5P8_Final_cpp(digest, ctx);
! 1132: }
! 1133:
! 1134: } // "C"
! 1135:
! 1136: #endif /* HAVE_SIMD */
! 1137: #endif /* __cplusplus */
! 1138: #endif /* __x86_64__ */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to simd-md5-parallel-x86_64.cpp CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / rsync