"Fossies" - the Fresh Open Source Software Archive

Member "unrar/blake2s_sse.cpp" (4 May 2022, 5115 Bytes) of package /linux/misc/unrarsrc-6.1.7.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "blake2s_sse.cpp" see the Fossies "Dox" file reference documentation.

    1 // Based on public domain code written in 2012 by Samuel Neves
    2 
    3 extern const byte blake2s_sigma[10][16];
    4 
    5 // Initialization vector.
    6 static __m128i blake2s_IV_0_3, blake2s_IV_4_7;
    7 
    8 #ifdef _WIN_64
    9 // Constants for cyclic rotation. Used in 64-bit mode in mm_rotr_epi32 macro.
   10 static __m128i crotr8, crotr16;
   11 #endif
   12 
   13 static void blake2s_init_sse()
   14 {
   15   // We cannot initialize these 128 bit variables in place when declaring
   16   // them globally, because global scope initialization is performed before
   17   // our SSE check and it would make code incompatible with older non-SSE2
   18   // CPUs. Also we cannot initialize them as static inside of function
   19   // using these variables, because SSE static initialization is not thread
   20   // safe: first thread starts initialization and sets "init done" flag even
   21   // if it is not done yet, second thread can attempt to access half-init
   22   // SSE data. So we moved init code here.
   23 
   24   blake2s_IV_0_3 = _mm_setr_epi32( 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A );
   25   blake2s_IV_4_7 = _mm_setr_epi32( 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 );
   26 
   27 #ifdef _WIN_64
   28   crotr8 = _mm_set_epi8( 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1 );
   29   crotr16 = _mm_set_epi8( 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 );
   30 #endif
   31 }
   32 
   33 
   34 #define LOAD(p)  _mm_load_si128( (__m128i *)(p) )
   35 #define STORE(p,r) _mm_store_si128((__m128i *)(p), r)
   36 
   37 #ifdef _WIN_32
   38 // 32-bit mode has less SSE2 registers and in MSVC2008 it is more efficient
   39 // to not use _mm_shuffle_epi8 here.
   40 #define mm_rotr_epi32(r, c) ( \
   41               _mm_xor_si128(_mm_srli_epi32( (r), c ),_mm_slli_epi32( (r), 32-c )) )
   42 #else
   43 #define mm_rotr_epi32(r, c) ( \
   44                 c==8 ? _mm_shuffle_epi8(r,crotr8) \
   45               : c==16 ? _mm_shuffle_epi8(r,crotr16) \
   46               : _mm_xor_si128(_mm_srli_epi32( (r), c ),_mm_slli_epi32( (r), 32-c )) )
   47 #endif
   48 
   49 
   50 #define G1(row1,row2,row3,row4,buf) \
   51   row1 = _mm_add_epi32( _mm_add_epi32( row1, buf), row2 ); \
   52   row4 = _mm_xor_si128( row4, row1 ); \
   53   row4 =  mm_rotr_epi32(row4, 16); \
   54   row3 = _mm_add_epi32( row3, row4 );   \
   55   row2 = _mm_xor_si128( row2, row3 ); \
   56   row2 =  mm_rotr_epi32(row2, 12);
   57 
   58 #define G2(row1,row2,row3,row4,buf) \
   59   row1 = _mm_add_epi32( _mm_add_epi32( row1, buf), row2 ); \
   60   row4 = _mm_xor_si128( row4, row1 ); \
   61   row4 =  mm_rotr_epi32(row4, 8); \
   62   row3 = _mm_add_epi32( row3, row4 );   \
   63   row2 = _mm_xor_si128( row2, row3 ); \
   64   row2 =  mm_rotr_epi32(row2, 7);
   65 
   66 #define DIAGONALIZE(row1,row2,row3,row4) \
   67   row4 = _mm_shuffle_epi32( row4, _MM_SHUFFLE(2,1,0,3) ); \
   68   row3 = _mm_shuffle_epi32( row3, _MM_SHUFFLE(1,0,3,2) ); \
   69   row2 = _mm_shuffle_epi32( row2, _MM_SHUFFLE(0,3,2,1) );
   70 
   71 #define UNDIAGONALIZE(row1,row2,row3,row4) \
   72   row4 = _mm_shuffle_epi32( row4, _MM_SHUFFLE(0,3,2,1) ); \
   73   row3 = _mm_shuffle_epi32( row3, _MM_SHUFFLE(1,0,3,2) ); \
   74   row2 = _mm_shuffle_epi32( row2, _MM_SHUFFLE(2,1,0,3) );
   75 
   76 #ifdef _WIN_64
   77   // MSVC 2008 in x64 mode expands _mm_set_epi32 to store to stack and load
   78   // from stack operations, which are slower than this code.
   79   #define _mm_set_epi32(i3,i2,i1,i0) \
   80     _mm_unpacklo_epi32(_mm_unpacklo_epi32(_mm_cvtsi32_si128(i0),_mm_cvtsi32_si128(i2)), \
   81                        _mm_unpacklo_epi32(_mm_cvtsi32_si128(i1),_mm_cvtsi32_si128(i3)))
   82 #endif
   83 
   84 // Original BLAKE2 SSE4.1 message loading code was a little slower in x86 mode
   85 // and about the same in x64 mode in our test. Perhaps depends on compiler.
   86 // We also tried _mm_i32gather_epi32 and _mm256_i32gather_epi32 AVX2 gather
   87 // instructions here, but they did not show any speed gain on i7-6700K.
   88 #define SSE_ROUND(m,row,r) \
   89 { \
   90   __m128i buf; \
   91   buf=_mm_set_epi32(m[blake2s_sigma[r][6]],m[blake2s_sigma[r][4]],m[blake2s_sigma[r][2]],m[blake2s_sigma[r][0]]); \
   92   G1(row[0],row[1],row[2],row[3],buf); \
   93   buf=_mm_set_epi32(m[blake2s_sigma[r][7]],m[blake2s_sigma[r][5]],m[blake2s_sigma[r][3]],m[blake2s_sigma[r][1]]); \
   94   G2(row[0],row[1],row[2],row[3],buf); \
   95   DIAGONALIZE(row[0],row[1],row[2],row[3]); \
   96   buf=_mm_set_epi32(m[blake2s_sigma[r][14]],m[blake2s_sigma[r][12]],m[blake2s_sigma[r][10]],m[blake2s_sigma[r][8]]); \
   97   G1(row[0],row[1],row[2],row[3],buf); \
   98   buf=_mm_set_epi32(m[blake2s_sigma[r][15]],m[blake2s_sigma[r][13]],m[blake2s_sigma[r][11]],m[blake2s_sigma[r][9]]); \
   99   G2(row[0],row[1],row[2],row[3],buf); \
  100   UNDIAGONALIZE(row[0],row[1],row[2],row[3]); \
  101 }
  102 
  103 
  104 static int blake2s_compress_sse( blake2s_state *S, const byte block[BLAKE2S_BLOCKBYTES] )
  105 {
  106   __m128i row[4];
  107   __m128i ff0, ff1;
  108   
  109   const uint32  *m = ( uint32 * )block;
  110 
  111   row[0] = ff0 = LOAD( &S->h[0] );
  112   row[1] = ff1 = LOAD( &S->h[4] );
  113 
  114   row[2] = blake2s_IV_0_3;
  115   row[3] = _mm_xor_si128( blake2s_IV_4_7, LOAD( &S->t[0] ) );
  116   SSE_ROUND( m, row, 0 );
  117   SSE_ROUND( m, row, 1 );
  118   SSE_ROUND( m, row, 2 );
  119   SSE_ROUND( m, row, 3 );
  120   SSE_ROUND( m, row, 4 );
  121   SSE_ROUND( m, row, 5 );
  122   SSE_ROUND( m, row, 6 );
  123   SSE_ROUND( m, row, 7 );
  124   SSE_ROUND( m, row, 8 );
  125   SSE_ROUND( m, row, 9 );
  126   STORE( &S->h[0], _mm_xor_si128( ff0, _mm_xor_si128( row[0], row[2] ) ) );
  127   STORE( &S->h[4], _mm_xor_si128( ff1, _mm_xor_si128( row[1], row[3] ) ) );
  128   return 0;
  129 }