ip7z/7zip
1
0
Fork 0
mirror of https://github.com/ip7z/7zip.git synced 2025-12-06 07:12:00 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity
7zip/C/Blake2s.c
Igor Pavlov 89a73b9012 24.06
2024-05-27 12:41:10 +05:00

2673 lines
77 KiB
C
Raw Permalink Blame History

/* Blake2s.c -- BLAKE2sp Hash
2024-05-18 : Igor Pavlov : Public domain
2015-2019 : Samuel Neves : original code : CC0 1.0 Universal (CC0 1.0). */
#include "Precomp.h"
// #include <stdio.h>
#include <string.h>
#include "Blake2.h"
#include "RotateDefs.h"
#include "Compiler.h"
#include "CpuArch.h"
/*
if defined(__AVX512F__) && defined(__AVX512VL__)
{
we define Z7_BLAKE2S_USE_AVX512_ALWAYS,
but the compiler can use avx512 for any code.
}
else if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS)
{ we use avx512 only for sse* and avx* branches of code. }
*/
// #define Z7_BLAKE2S_USE_AVX512_ALWAYS // for debug
#if defined(__SSE2__)
#define Z7_BLAKE2S_USE_VECTORS
#elif defined(MY_CPU_X86_OR_AMD64)
#if defined(_MSC_VER) && _MSC_VER > 1200 \
|| defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \
|| defined(__clang__) \
|| defined(__INTEL_COMPILER)
#define Z7_BLAKE2S_USE_VECTORS
#endif
#endif
#ifdef Z7_BLAKE2S_USE_VECTORS
#define Z7_BLAKE2SP_USE_FUNCTIONS
// define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED, if CBlake2sp can be non aligned for 32-bytes.
// #define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED
// SSSE3 : for _mm_shuffle_epi8 (pshufb) that improves the performance for 5-15%.
#if defined(__SSSE3__)
#define Z7_BLAKE2S_USE_SSSE3
#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \
|| defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \
|| defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \
|| defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \
|| defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000)
#define Z7_BLAKE2S_USE_SSSE3
#endif
#ifdef Z7_BLAKE2S_USE_SSSE3
/* SSE41 : for _mm_insert_epi32 (pinsrd)
it can slightly reduce code size and improves the performance in some cases.
it's used only for last 512-1024 bytes, if FAST versions (2 or 3) of vector algos are used.
it can be used for all blocks in another algos (4+).
*/
#if defined(__SSE4_1__)
#define Z7_BLAKE2S_USE_SSE41
#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \
|| defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \
|| defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \
|| defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \
|| defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000)
#define Z7_BLAKE2S_USE_SSE41
#endif
#endif // SSSE3
#if defined(__GNUC__) || defined(__clang__)
#if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) && !(defined(__AVX512F__) && defined(__AVX512VL__))
#define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("avx512vl,avx512f")))
#else
#if defined(Z7_BLAKE2S_USE_SSE41)
#define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse4.1")))
#elif defined(Z7_BLAKE2S_USE_SSSE3)
#define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("ssse3")))
#else
#define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse2")))
#endif
#endif
#endif
#if defined(__AVX2__)
#define Z7_BLAKE2S_USE_AVX2
#else
#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \
|| defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \
|| defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100)
#define Z7_BLAKE2S_USE_AVX2
#ifdef Z7_BLAKE2S_USE_AVX2
#if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) && !(defined(__AVX512F__) && defined(__AVX512VL__))
#define BLAKE2S_ATTRIB_AVX2 __attribute__((__target__("avx512vl,avx512f")))
#else
#define BLAKE2S_ATTRIB_AVX2 __attribute__((__target__("avx2")))
#endif
#endif
#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \
|| defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400)
#if (Z7_MSC_VER_ORIGINAL == 1900)
#pragma warning(disable : 4752) // found Intel(R) Advanced Vector Extensions; consider using /arch:AVX
#endif
#define Z7_BLAKE2S_USE_AVX2
#endif
#endif
#ifdef Z7_BLAKE2S_USE_SSE41
#include <smmintrin.h> // SSE4.1
#elif defined(Z7_BLAKE2S_USE_SSSE3)
#include <tmmintrin.h> // SSSE3
#else
#include <emmintrin.h> // SSE2
#endif
#ifdef Z7_BLAKE2S_USE_AVX2
#include <immintrin.h>
#if defined(__clang__)
#include <avxintrin.h>
#include <avx2intrin.h>
#endif
#endif // avx2
#if defined(__AVX512F__) && defined(__AVX512VL__)
// && defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL > 1930)
#ifndef Z7_BLAKE2S_USE_AVX512_ALWAYS
#define Z7_BLAKE2S_USE_AVX512_ALWAYS
#endif
// #pragma message ("=== Blake2s AVX512")
#endif
#define Z7_BLAKE2S_USE_V128_FAST
// for speed optimization for small messages:
// #define Z7_BLAKE2S_USE_V128_WAY2
#ifdef Z7_BLAKE2S_USE_AVX2
// for debug:
// gather is slow
// #define Z7_BLAKE2S_USE_GATHER
#define Z7_BLAKE2S_USE_AVX2_FAST
// for speed optimization for small messages:
// #define Z7_BLAKE2S_USE_AVX2_WAY2
// #define Z7_BLAKE2S_USE_AVX2_WAY4
#if defined(Z7_BLAKE2S_USE_AVX2_WAY2) || \
defined(Z7_BLAKE2S_USE_AVX2_WAY4)
#define Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#endif
#endif
#define Z7_BLAKE2SP_ALGO_DEFAULT 0
#define Z7_BLAKE2SP_ALGO_SCALAR 1
#ifdef Z7_BLAKE2S_USE_V128_FAST
#define Z7_BLAKE2SP_ALGO_V128_FAST 2
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_FAST
#define Z7_BLAKE2SP_ALGO_V256_FAST 3
#endif
#define Z7_BLAKE2SP_ALGO_V128_WAY1 4
#ifdef Z7_BLAKE2S_USE_V128_WAY2
#define Z7_BLAKE2SP_ALGO_V128_WAY2 5
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY2
#define Z7_BLAKE2SP_ALGO_V256_WAY2 6
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY4
#define Z7_BLAKE2SP_ALGO_V256_WAY4 7
#endif
#endif // Z7_BLAKE2S_USE_VECTORS
#define BLAKE2S_FINAL_FLAG (~(UInt32)0)
#define NSW Z7_BLAKE2SP_NUM_STRUCT_WORDS
#define SUPER_BLOCK_SIZE (Z7_BLAKE2S_BLOCK_SIZE * Z7_BLAKE2SP_PARALLEL_DEGREE)
#define SUPER_BLOCK_MASK (SUPER_BLOCK_SIZE - 1)
#define V_INDEX_0_0 0
#define V_INDEX_1_0 1
#define V_INDEX_2_0 2
#define V_INDEX_3_0 3
#define V_INDEX_0_1 4
#define V_INDEX_1_1 5
#define V_INDEX_2_1 6
#define V_INDEX_3_1 7
#define V_INDEX_0_2 8
#define V_INDEX_1_2 9
#define V_INDEX_2_2 10
#define V_INDEX_3_2 11
#define V_INDEX_0_3 12
#define V_INDEX_1_3 13
#define V_INDEX_2_3 14
#define V_INDEX_3_3 15
#define V_INDEX_4_0 0
#define V_INDEX_5_0 1
#define V_INDEX_6_0 2
#define V_INDEX_7_0 3
#define V_INDEX_7_1 4
#define V_INDEX_4_1 5
#define V_INDEX_5_1 6
#define V_INDEX_6_1 7
#define V_INDEX_6_2 8
#define V_INDEX_7_2 9
#define V_INDEX_4_2 10
#define V_INDEX_5_2 11
#define V_INDEX_5_3 12
#define V_INDEX_6_3 13
#define V_INDEX_7_3 14
#define V_INDEX_4_3 15
#define V(row, col) v[V_INDEX_ ## row ## _ ## col]
#define k_Blake2s_IV_0 0x6A09E667UL
#define k_Blake2s_IV_1 0xBB67AE85UL
#define k_Blake2s_IV_2 0x3C6EF372UL
#define k_Blake2s_IV_3 0xA54FF53AUL
#define k_Blake2s_IV_4 0x510E527FUL
#define k_Blake2s_IV_5 0x9B05688CUL
#define k_Blake2s_IV_6 0x1F83D9ABUL
#define k_Blake2s_IV_7 0x5BE0CD19UL
#define KIV(n) (k_Blake2s_IV_## n)
#ifdef Z7_BLAKE2S_USE_VECTORS
MY_ALIGN(16)
static const UInt32 k_Blake2s_IV[8] =
{
KIV(0), KIV(1), KIV(2), KIV(3), KIV(4), KIV(5), KIV(6), KIV(7)
};
#endif
#define STATE_T(s) ((s) + 8)
#define STATE_F(s) ((s) + 10)
#ifdef Z7_BLAKE2S_USE_VECTORS
#define LOAD_128(p) _mm_load_si128 ((const __m128i *)(const void *)(p))
#define LOADU_128(p) _mm_loadu_si128((const __m128i *)(const void *)(p))
#ifdef Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED
// here we use unaligned load and stores
// use this branch if CBlake2sp can be unaligned for 16 bytes
#define STOREU_128(p, r) _mm_storeu_si128((__m128i *)(void *)(p), r)
#define LOAD_128_FROM_STRUCT(p) LOADU_128(p)
#define STORE_128_TO_STRUCT(p, r) STOREU_128(p, r)
#else
// here we use aligned load and stores
// use this branch if CBlake2sp is aligned for 16 bytes
#define STORE_128(p, r) _mm_store_si128((__m128i *)(void *)(p), r)
#define LOAD_128_FROM_STRUCT(p) LOAD_128(p)
#define STORE_128_TO_STRUCT(p, r) STORE_128(p, r)
#endif
#endif // Z7_BLAKE2S_USE_VECTORS
#if 0
static void PrintState(const UInt32 *s, unsigned num)
{
unsigned i;
printf("\n");
for (i = 0; i < num; i++)
printf(" %08x", (unsigned)s[i]);
}
static void PrintStates2(const UInt32 *s, unsigned x, unsigned y)
{
unsigned i;
for (i = 0; i < y; i++)
PrintState(s + i * x, x);
printf("\n");
}
#endif
#define REP8_MACRO(m) { m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) }
#define BLAKE2S_NUM_ROUNDS 10
#if defined(Z7_BLAKE2S_USE_VECTORS)
#define ROUNDS_LOOP(mac) \
{ unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r++) mac(r) }
#endif
/*
#define ROUNDS_LOOP_2(mac) \
{ unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r += 2) { mac(r) mac(r + 1) } }
*/
#if 0 || 1 && !defined(Z7_BLAKE2S_USE_VECTORS)
#define ROUNDS_LOOP_UNROLLED(m) \
{ m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) }
#endif
#define SIGMA_TABLE(M) \
M( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ), \
M( 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 ), \
M( 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 ), \
M( 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 ), \
M( 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 ), \
M( 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 ), \
M( 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 ), \
M( 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 ), \
M( 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 ), \
M( 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 )
#define SIGMA_TABLE_MULT(m, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
{ a0*m,a1*m,a2*m,a3*m,a4*m,a5*m,a6*m,a7*m,a8*m,a9*m,a10*m,a11*m,a12*m,a13*m,a14*m,a15*m }
#define SIGMA_TABLE_MULT_4( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
SIGMA_TABLE_MULT(4, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15)
// MY_ALIGN(32)
MY_ALIGN(16)
static const Byte k_Blake2s_Sigma_4[BLAKE2S_NUM_ROUNDS][16] =
{ SIGMA_TABLE(SIGMA_TABLE_MULT_4) };
#define GET_SIGMA_PTR(p, index) \
((const void *)((const Byte *)(const void *)(p) + (index)))
#define GET_STATE_TABLE_PTR_FROM_BYTE_POS(s, pos) \
((UInt32 *)(void *)((Byte *)(void *)(s) + (pos)))
#ifdef Z7_BLAKE2S_USE_VECTORS
#if 0
// use loading constants from memory
// is faster for some compilers.
#define KK4(n) KIV(n), KIV(n), KIV(n), KIV(n)
MY_ALIGN(64)
static const UInt32 k_Blake2s_IV_WAY4[]=
{
KK4(0), KK4(1), KK4(2), KK4(3), KK4(4), KK4(5), KK4(6), KK4(7)
};
#define GET_128_IV_WAY4(i) LOAD_128(k_Blake2s_IV_WAY4 + 4 * (i))
#else
// use constant generation:
#define GET_128_IV_WAY4(i) _mm_set1_epi32((Int32)KIV(i))
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#define GET_CONST_128_FROM_ARRAY32(k) \
_mm_set_epi32((Int32)(k)[3], (Int32)(k)[2], (Int32)(k)[1], (Int32)(k)[0])
#endif
#if 0
#define k_r8 _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)
#define k_r16 _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)
#define k_inc _mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE)
#define k_iv0_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0)
#define k_iv4_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4)
#else
#if defined(Z7_BLAKE2S_USE_SSSE3) && \
!defined(Z7_BLAKE2S_USE_AVX512_ALWAYS)
MY_ALIGN(16) static const Byte k_r8_arr [16] = { 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12 };
MY_ALIGN(16) static const Byte k_r16_arr[16] = { 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 };
#define k_r8 LOAD_128(k_r8_arr)
#define k_r16 LOAD_128(k_r16_arr)
#endif
MY_ALIGN(16) static const UInt32 k_inc_arr[4] = { Z7_BLAKE2S_BLOCK_SIZE, 0, 0, 0 };
#define k_inc LOAD_128(k_inc_arr)
#define k_iv0_128 LOAD_128(k_Blake2s_IV + 0)
#define k_iv4_128 LOAD_128(k_Blake2s_IV + 4)
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#ifdef Z7_BLAKE2S_USE_AVX2
#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION < 80000)
#define MY_mm256_set_m128i(hi, lo) _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1)
#else
#define MY_mm256_set_m128i _mm256_set_m128i
#endif
#define SET_FROM_128(a) MY_mm256_set_m128i(a, a)
#ifndef Z7_BLAKE2S_USE_AVX512_ALWAYS
MY_ALIGN(32) static const Byte k_r8_arr_256 [32] =
{
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12,
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12
};
MY_ALIGN(32) static const Byte k_r16_arr_256[32] =
{
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13,
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13
};
#define k_r8_256 LOAD_256(k_r8_arr_256)
#define k_r16_256 LOAD_256(k_r16_arr_256)
#endif
// #define k_r8_256 SET_FROM_128(_mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1))
// #define k_r16_256 SET_FROM_128(_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2))
// #define k_inc_256 SET_FROM_128(_mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE))
// #define k_iv0_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0))
#define k_iv4_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4))
#endif // Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#endif
/*
IPC(TP) ports:
1 p__5 : skl- : SSE : shufps : _mm_shuffle_ps
2 p_15 : icl+
1 p__5 : nhm-bdw : SSE : xorps : _mm_xor_ps
3 p015 : skl+
3 p015 : SSE2 : pxor : _mm_xor_si128
2 p_15: snb-bdw : SSE2 : padd : _mm_add_epi32
2 p0_5: mrm-wsm :
3 p015 : skl+
2 p_15 : ivb-,icl+ : SSE2 : punpcklqdq, punpckhqdq, punpckldq, punpckhdq
2 p_15 : : SSE2 : pshufd : _mm_shuffle_epi32
2 p_15 : : SSE2 : pshuflw : _mm_shufflelo_epi16
2 p_15 : : SSE2 : psrldq :
2 p_15 : : SSE3 : pshufb : _mm_shuffle_epi8
2 p_15 : : SSE4 : pblendw : _mm_blend_epi16
1 p__5 : hsw-skl : *
1 p0 : SSE2 : pslld (i8) : _mm_slli_si128
2 p01 : skl+ :
2 p_15 : ivb- : SSE3 : palignr
1 p__5 : hsw+
2 p_15 + p23 : ivb-, icl+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, m32, i8)
1 p__5 + p23 : hsw-skl
1 p_15 + p5 : ivb-, ice+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, r32, i8)
0.5 2*p5 : hsw-skl
2 p23 : SSE2 : movd (m32)
3 p23A : adl :
1 p5: : SSE2 : movd (r32)
*/
#if 0 && defined(__XOP__)
// we must debug and test __XOP__ instruction
#include <x86intrin.h>
#include <ammintrin.h>
#define LOAD_ROTATE_CONSTS
#define MM_ROR_EPI32(r, c) _mm_roti_epi32(r, -(c))
#define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED
#elif 1 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS)
#define LOAD_ROTATE_CONSTS
#define MM_ROR_EPI32(r, c) _mm_ror_epi32(r, c)
#define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED
#else
// MSVC_1937+ uses "orps" instruction for _mm_or_si128().
// But "orps" has low throughput: TP=1 for bdw-nhm.
// So it can be better to use _mm_add_epi32()/"paddd" (TP=2 for bdw-nhm) instead of "xorps".
// But "orps" is fast for modern cpus (skl+).
// So we are default with "or" version:
#if 0 || 0 && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL > 1937
// minor optimization for some old cpus, if "xorps" is slow.
#define MM128_EPI32_OR_or_ADD _mm_add_epi32
#else
#define MM128_EPI32_OR_or_ADD _mm_or_si128
#endif
#define MM_ROR_EPI32_VIA_SHIFT(r, c)( \
MM128_EPI32_OR_or_ADD( \
_mm_srli_epi32((r), (c)), \
_mm_slli_epi32((r), 32-(c))))
#if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41)
#define LOAD_ROTATE_CONSTS \
const __m128i r8 = k_r8; \
const __m128i r16 = k_r16;
#define MM_ROR_EPI32(r, c) ( \
( 8==(c)) ? _mm_shuffle_epi8(r,r8) \
: (16==(c)) ? _mm_shuffle_epi8(r,r16) \
: MM_ROR_EPI32_VIA_SHIFT(r, c))
#else
#define LOAD_ROTATE_CONSTS
#define MM_ROR_EPI32(r, c) ( \
(16==(c)) ? _mm_shufflehi_epi16(_mm_shufflelo_epi16(r, 0xb1), 0xb1) \
: MM_ROR_EPI32_VIA_SHIFT(r, c))
#endif
#endif
/*
we have 3 main ways to load 4 32-bit integers to __m128i:
1) SSE2: _mm_set_epi32()
2) SSE2: _mm_unpacklo_epi64() / _mm_unpacklo_epi32 / _mm_cvtsi32_si128()
3) SSE41: _mm_insert_epi32() and _mm_cvtsi32_si128()
good compiler for _mm_set_epi32() generates these instructions:
{
movd xmm, [m32]; vpunpckldq; vpunpckldq; vpunpcklqdq;
}
good new compiler generates one instruction
{
for _mm_insert_epi32() : { pinsrd xmm, [m32], i }
for _mm_cvtsi32_si128() : { movd xmm, [m32] }
}
but vc2010 generates slow pair of instructions:
{
for _mm_insert_epi32() : { mov r32, [m32]; pinsrd xmm, r32, i }
for _mm_cvtsi32_si128() : { mov r32, [m32]; movd xmm, r32 }
}
_mm_insert_epi32() (pinsrd) code reduces xmm register pressure
in comparison with _mm_set_epi32() (movd + vpunpckld) code.
Note that variant with "movd xmm, r32" can be more slow,
but register pressure can be more important.
So we can force to "pinsrd" always.
*/
// #if !defined(Z7_MSC_VER_ORIGINAL) || Z7_MSC_VER_ORIGINAL > 1600 || defined(MY_CPU_X86)
#ifdef Z7_BLAKE2S_USE_SSE41
/* _mm_set_epi32() can be more effective for GCC and CLANG
_mm_insert_epi32() is more effective for MSVC */
#if 0 || 1 && defined(Z7_MSC_VER_ORIGINAL)
#define Z7_BLAKE2S_USE_INSERT_INSTRUCTION
#endif
#endif // USE_SSE41
// #endif
#ifdef Z7_BLAKE2S_USE_INSERT_INSTRUCTION
// for SSE4.1
#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \
_mm_insert_epi32( \
_mm_insert_epi32( \
_mm_insert_epi32( \
_mm_cvtsi32_si128( \
*(const Int32 *)p0), \
*(const Int32 *)p1, 1), \
*(const Int32 *)p2, 2), \
*(const Int32 *)p3, 3)
#elif 0 || 1 && defined(Z7_MSC_VER_ORIGINAL)
/* MSVC 1400 implements _mm_set_epi32() via slow memory write/read.
Also _mm_unpacklo_epi32 is more effective for another MSVC compilers.
But _mm_set_epi32() is more effective for GCC and CLANG.
So we use _mm_unpacklo_epi32 for MSVC only */
#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \
_mm_unpacklo_epi64( \
_mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p0), \
_mm_cvtsi32_si128(*(const Int32 *)p1)), \
_mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p2), \
_mm_cvtsi32_si128(*(const Int32 *)p3)))
#else
#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \
_mm_set_epi32( \
*(const Int32 *)p3, \
*(const Int32 *)p2, \
*(const Int32 *)p1, \
*(const Int32 *)p0)
#endif
#define SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3) \
MM_LOAD_EPI32_FROM_4_POINTERS( \
GET_SIGMA_PTR(input, i0), \
GET_SIGMA_PTR(input, i1), \
GET_SIGMA_PTR(input, i2), \
GET_SIGMA_PTR(input, i3))
#define SET_ROW_FROM_SIGMA(input, sigma_index) \
SET_ROW_FROM_SIGMA_BASE(input, \
sigma[(sigma_index) ], \
sigma[(sigma_index) + 2 * 1], \
sigma[(sigma_index) + 2 * 2], \
sigma[(sigma_index) + 2 * 3]) \
#define ADD_128(a, b) _mm_add_epi32(a, b)
#define XOR_128(a, b) _mm_xor_si128(a, b)
#define D_ADD_128(dest, src) dest = ADD_128(dest, src)
#define D_XOR_128(dest, src) dest = XOR_128(dest, src)
#define D_ROR_128(dest, shift) dest = MM_ROR_EPI32(dest, shift)
#define D_ADD_EPI64_128(dest, src) dest = _mm_add_epi64(dest, src)
#define AXR(a, b, d, shift) \
D_ADD_128(a, b); \
D_XOR_128(d, a); \
D_ROR_128(d, shift);
#define AXR2(a, b, c, d, input, sigma_index, shift1, shift2) \
a = _mm_add_epi32 (a, SET_ROW_FROM_SIGMA(input, sigma_index)); \
AXR(a, b, d, shift1) \
AXR(c, d, b, shift2)
#define ROTATE_WORDS_TO_RIGHT(a, n) \
a = _mm_shuffle_epi32(a, _MM_SHUFFLE((3+n)&3, (2+n)&3, (1+n)&3, (0+n)&3));
#define AXR4(a, b, c, d, input, sigma_index) \
AXR2(a, b, c, d, input, sigma_index, 16, 12) \
AXR2(a, b, c, d, input, sigma_index + 1, 8, 7) \
#define RR2(a, b, c, d, input) \
{ \
AXR4(a, b, c, d, input, 0) \
ROTATE_WORDS_TO_RIGHT(b, 1) \
ROTATE_WORDS_TO_RIGHT(c, 2) \
ROTATE_WORDS_TO_RIGHT(d, 3) \
AXR4(a, b, c, d, input, 8) \
ROTATE_WORDS_TO_RIGHT(b, 3) \
ROTATE_WORDS_TO_RIGHT(c, 2) \
ROTATE_WORDS_TO_RIGHT(d, 1) \
}
/*
Way1:
per 64 bytes block:
10 rounds * 4 iters * (7 + 2) = 360 cycles = if pslld TP=1
* (7 + 1) = 320 cycles = if pslld TP=2 (skl+)
additional operations per 7_op_iter :
4 movzx byte mem
1 movd mem
3 pinsrd mem
1.5 pshufd
*/
static
#if 0 || 0 && (defined(Z7_BLAKE2S_USE_V128_WAY2) || \
defined(Z7_BLAKE2S_USE_V256_WAY2))
Z7_NO_INLINE
#else
Z7_FORCE_INLINE
#endif
#ifdef BLAKE2S_ATTRIB_128BIT
BLAKE2S_ATTRIB_128BIT
#endif
void
Z7_FASTCALL
Blake2s_Compress_V128_Way1(UInt32 * const s, const Byte * const input)
{
__m128i a, b, c, d;
__m128i f0, f1;
LOAD_ROTATE_CONSTS
d = LOAD_128_FROM_STRUCT(STATE_T(s));
c = k_iv0_128;
a = f0 = LOAD_128_FROM_STRUCT(s);
b = f1 = LOAD_128_FROM_STRUCT(s + 4);
D_ADD_EPI64_128(d, k_inc);
STORE_128_TO_STRUCT (STATE_T(s), d);
D_XOR_128(d, k_iv4_128);
#define RR(r) { const Byte * const sigma = k_Blake2s_Sigma_4[r]; \
RR2(a, b, c, d, input) }
ROUNDS_LOOP(RR)
#undef RR
STORE_128_TO_STRUCT(s , XOR_128(f0, XOR_128(a, c)));
STORE_128_TO_STRUCT(s + 4, XOR_128(f1, XOR_128(b, d)));
}
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_128BIT
BLAKE2S_ATTRIB_128BIT
#endif
void
Z7_FASTCALL
Blake2sp_Compress2_V128_Way1(UInt32 *s_items, const Byte *data, const Byte *end)
{
size_t pos = 0;
do
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
Blake2s_Compress_V128_Way1(s, data);
data += Z7_BLAKE2S_BLOCK_SIZE;
pos += Z7_BLAKE2S_BLOCK_SIZE;
pos &= SUPER_BLOCK_MASK;
}
while (data != end);
}
#if defined(Z7_BLAKE2S_USE_V128_WAY2) || \
defined(Z7_BLAKE2S_USE_AVX2_WAY2)
#if 1
#define Z7_BLAKE2S_CompressSingleBlock(s, data) \
Blake2sp_Compress2_V128_Way1(s, data, \
(const Byte *)(const void *)(data) + Z7_BLAKE2S_BLOCK_SIZE)
#else
#define Z7_BLAKE2S_CompressSingleBlock Blake2s_Compress_V128_Way1
#endif
#endif
#if (defined(Z7_BLAKE2S_USE_AVX2_WAY_SLOW) || \
defined(Z7_BLAKE2S_USE_V128_WAY2)) && \
!defined(Z7_BLAKE2S_USE_GATHER)
#define AXR2_LOAD_INDEXES(sigma_index) \
const unsigned i0 = sigma[(sigma_index)]; \
const unsigned i1 = sigma[(sigma_index) + 2 * 1]; \
const unsigned i2 = sigma[(sigma_index) + 2 * 2]; \
const unsigned i3 = sigma[(sigma_index) + 2 * 3]; \
#define SET_ROW_FROM_SIGMA_W(input) \
SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3)
#endif
#ifdef Z7_BLAKE2S_USE_V128_WAY2
#if 1 || !defined(Z7_BLAKE2S_USE_SSE41)
/* we use SET_ROW_FROM_SIGMA_BASE, that uses
(SSE4) _mm_insert_epi32(), if Z7_BLAKE2S_USE_INSERT_INSTRUCTION is defined
(SSE2) _mm_set_epi32()
MSVC can be faster for this branch:
*/
#define AXR2_W(sigma_index, shift1, shift2) \
{ \
AXR2_LOAD_INDEXES(sigma_index) \
a0 = _mm_add_epi32(a0, SET_ROW_FROM_SIGMA_W(data)); \
a1 = _mm_add_epi32(a1, SET_ROW_FROM_SIGMA_W(data + Z7_BLAKE2S_BLOCK_SIZE)); \
AXR(a0, b0, d0, shift1) \
AXR(a1, b1, d1, shift1) \
AXR(c0, d0, b0, shift2) \
AXR(c1, d1, b1, shift2) \
}
#else
/* we use interleaved _mm_insert_epi32():
GCC can be faster for this branch:
*/
#define AXR2_W_PRE_INSERT(sigma_index, i) \
{ const unsigned ii = sigma[(sigma_index) + i * 2]; \
t0 = _mm_insert_epi32(t0, *(const Int32 *)GET_SIGMA_PTR(data, ii), i); \
t1 = _mm_insert_epi32(t1, *(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii), i); \
}
#define AXR2_W(sigma_index, shift1, shift2) \
{ __m128i t0, t1; \
{ const unsigned ii = sigma[sigma_index]; \
t0 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, ii)); \
t1 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii)); \
} \
AXR2_W_PRE_INSERT(sigma_index, 1) \
AXR2_W_PRE_INSERT(sigma_index, 2) \
AXR2_W_PRE_INSERT(sigma_index, 3) \
a0 = _mm_add_epi32(a0, t0); \
a1 = _mm_add_epi32(a1, t1); \
AXR(a0, b0, d0, shift1) \
AXR(a1, b1, d1, shift1) \
AXR(c0, d0, b0, shift2) \
AXR(c1, d1, b1, shift2) \
}
#endif
#define AXR4_W(sigma_index) \
AXR2_W(sigma_index, 16, 12) \
AXR2_W(sigma_index + 1, 8, 7) \
#define WW(r) \
{ const Byte * const sigma = k_Blake2s_Sigma_4[r]; \
AXR4_W(0) \
ROTATE_WORDS_TO_RIGHT(b0, 1) \
ROTATE_WORDS_TO_RIGHT(b1, 1) \
ROTATE_WORDS_TO_RIGHT(c0, 2) \
ROTATE_WORDS_TO_RIGHT(c1, 2) \
ROTATE_WORDS_TO_RIGHT(d0, 3) \
ROTATE_WORDS_TO_RIGHT(d1, 3) \
AXR4_W(8) \
ROTATE_WORDS_TO_RIGHT(b0, 3) \
ROTATE_WORDS_TO_RIGHT(b1, 3) \
ROTATE_WORDS_TO_RIGHT(c0, 2) \
ROTATE_WORDS_TO_RIGHT(c1, 2) \
ROTATE_WORDS_TO_RIGHT(d0, 1) \
ROTATE_WORDS_TO_RIGHT(d1, 1) \
}
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_128BIT
BLAKE2S_ATTRIB_128BIT
#endif
void
Z7_FASTCALL
Blake2sp_Compress2_V128_Way2(UInt32 *s_items, const Byte *data, const Byte *end)
{
size_t pos = 0;
end -= Z7_BLAKE2S_BLOCK_SIZE;
if (data != end)
{
LOAD_ROTATE_CONSTS
do
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
__m128i a0, b0, c0, d0;
__m128i a1, b1, c1, d1;
{
const __m128i inc = k_inc;
const __m128i temp = k_iv4_128;
d0 = LOAD_128_FROM_STRUCT (STATE_T(s));
d1 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW));
D_ADD_EPI64_128(d0, inc);
D_ADD_EPI64_128(d1, inc);
STORE_128_TO_STRUCT (STATE_T(s ), d0);
STORE_128_TO_STRUCT (STATE_T(s + NSW), d1);
D_XOR_128(d0, temp);
D_XOR_128(d1, temp);
}
c1 = c0 = k_iv0_128;
a0 = LOAD_128_FROM_STRUCT(s);
b0 = LOAD_128_FROM_STRUCT(s + 4);
a1 = LOAD_128_FROM_STRUCT(s + NSW);
b1 = LOAD_128_FROM_STRUCT(s + NSW + 4);
ROUNDS_LOOP (WW)
#undef WW
D_XOR_128(a0, c0);
D_XOR_128(b0, d0);
D_XOR_128(a1, c1);
D_XOR_128(b1, d1);
D_XOR_128(a0, LOAD_128_FROM_STRUCT(s));
D_XOR_128(b0, LOAD_128_FROM_STRUCT(s + 4));
D_XOR_128(a1, LOAD_128_FROM_STRUCT(s + NSW));
D_XOR_128(b1, LOAD_128_FROM_STRUCT(s + NSW + 4));
STORE_128_TO_STRUCT(s, a0);
STORE_128_TO_STRUCT(s + 4, b0);
STORE_128_TO_STRUCT(s + NSW, a1);
STORE_128_TO_STRUCT(s + NSW + 4, b1);
data += Z7_BLAKE2S_BLOCK_SIZE * 2;
pos += Z7_BLAKE2S_BLOCK_SIZE * 2;
pos &= SUPER_BLOCK_MASK;
}
while (data < end);
if (data != end)
return;
}
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
Z7_BLAKE2S_CompressSingleBlock(s, data);
}
}
#endif // Z7_BLAKE2S_USE_V128_WAY2
#ifdef Z7_BLAKE2S_USE_V128_WAY2
#define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way2
#else
#define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way1
#endif
#ifdef Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED
#define ROT_128_8(x) MM_ROR_EPI32(x, 8)
#define ROT_128_16(x) MM_ROR_EPI32(x, 16)
#define ROT_128_7(x) MM_ROR_EPI32(x, 7)
#define ROT_128_12(x) MM_ROR_EPI32(x, 12)
#else
#if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41)
#define ROT_128_8(x) _mm_shuffle_epi8(x, r8) // k_r8
#define ROT_128_16(x) _mm_shuffle_epi8(x, r16) // k_r16
#else
#define ROT_128_8(x) MM_ROR_EPI32_VIA_SHIFT(x, 8)
#define ROT_128_16(x) MM_ROR_EPI32_VIA_SHIFT(x, 16)
#endif
#define ROT_128_7(x) MM_ROR_EPI32_VIA_SHIFT(x, 7)
#define ROT_128_12(x) MM_ROR_EPI32_VIA_SHIFT(x, 12)
#endif
#if 1
// this branch can provide similar speed on x86* in most cases,
// because [base + index*4] provides same speed as [base + index].
// but some compilers can generate different code with this branch, that can be faster sometimes.
// this branch uses additional table of 10*16=160 bytes.
#define SIGMA_TABLE_MULT_16( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
SIGMA_TABLE_MULT(16, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15)
MY_ALIGN(16)
static const Byte k_Blake2s_Sigma_16[BLAKE2S_NUM_ROUNDS][16] =
{ SIGMA_TABLE(SIGMA_TABLE_MULT_16) };
#define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_16[r];
#define GET_SIGMA_VAL_128(n) (sigma[n])
#else
#define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_4[r];
#define GET_SIGMA_VAL_128(n) (4 * (size_t)sigma[n])
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_FAST
#if 1
#define SIGMA_TABLE_MULT_32( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
SIGMA_TABLE_MULT(32, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15)
MY_ALIGN(64)
static const UInt16 k_Blake2s_Sigma_32[BLAKE2S_NUM_ROUNDS][16] =
{ SIGMA_TABLE(SIGMA_TABLE_MULT_32) };
#define GET_SIGMA_PTR_256(r) const UInt16 * const sigma = k_Blake2s_Sigma_32[r];
#define GET_SIGMA_VAL_256(n) (sigma[n])
#else
#define GET_SIGMA_PTR_256(r) const Byte * const sigma = k_Blake2s_Sigma_4[r];
#define GET_SIGMA_VAL_256(n) (8 * (size_t)sigma[n])
#endif
#endif // Z7_BLAKE2S_USE_AVX2_FAST
#define D_ROT_128_7(dest) dest = ROT_128_7(dest)
#define D_ROT_128_8(dest) dest = ROT_128_8(dest)
#define D_ROT_128_12(dest) dest = ROT_128_12(dest)
#define D_ROT_128_16(dest) dest = ROT_128_16(dest)
#define OP_L(a, i) D_ADD_128 (V(a, 0), \
LOAD_128((const Byte *)(w) + GET_SIGMA_VAL_128(2*(a)+(i))));
#define OP_0(a) OP_L(a, 0)
#define OP_7(a) OP_L(a, 1)
#define OP_1(a) D_ADD_128 (V(a, 0), V(a, 1));
#define OP_2(a) D_XOR_128 (V(a, 3), V(a, 0));
#define OP_4(a) D_ADD_128 (V(a, 2), V(a, 3));
#define OP_5(a) D_XOR_128 (V(a, 1), V(a, 2));
#define OP_3(a) D_ROT_128_16 (V(a, 3));
#define OP_6(a) D_ROT_128_12 (V(a, 1));
#define OP_8(a) D_ROT_128_8 (V(a, 3));
#define OP_9(a) D_ROT_128_7 (V(a, 1));
// for 32-bit x86 : interleave mode works slower, because of register pressure.
#if 0 || 1 && (defined(MY_CPU_X86) \
|| defined(__GNUC__) && !defined(__clang__))
// non-inteleaved version:
// is fast for x86 32-bit.
// is fast for GCC x86-64.
#define V4G(a) \
OP_0 (a) \
OP_1 (a) \
OP_2 (a) \
OP_3 (a) \
OP_4 (a) \
OP_5 (a) \
OP_6 (a) \
OP_7 (a) \
OP_1 (a) \
OP_2 (a) \
OP_8 (a) \
OP_4 (a) \
OP_5 (a) \
OP_9 (a) \
#define V4R \
{ \
V4G (0) \
V4G (1) \
V4G (2) \
V4G (3) \
V4G (4) \
V4G (5) \
V4G (6) \
V4G (7) \
}
#elif 0 || 1 && defined(MY_CPU_X86)
#define OP_INTER_2(op, a,b) \
op (a) \
op (b) \
#define V4G(a,b) \
OP_INTER_2 (OP_0, a,b) \
OP_INTER_2 (OP_1, a,b) \
OP_INTER_2 (OP_2, a,b) \
OP_INTER_2 (OP_3, a,b) \
OP_INTER_2 (OP_4, a,b) \
OP_INTER_2 (OP_5, a,b) \
OP_INTER_2 (OP_6, a,b) \
OP_INTER_2 (OP_7, a,b) \
OP_INTER_2 (OP_1, a,b) \
OP_INTER_2 (OP_2, a,b) \
OP_INTER_2 (OP_8, a,b) \
OP_INTER_2 (OP_4, a,b) \
OP_INTER_2 (OP_5, a,b) \
OP_INTER_2 (OP_9, a,b) \
#define V4R \
{ \
V4G (0, 1) \
V4G (2, 3) \
V4G (4, 5) \
V4G (6, 7) \
}
#else
// iterleave-4 version is fast for x64 (MSVC/CLANG)
#define OP_INTER_4(op, a,b,c,d) \
op (a) \
op (b) \
op (c) \
op (d) \
#define V4G(a,b,c,d) \
OP_INTER_4 (OP_0, a,b,c,d) \
OP_INTER_4 (OP_1, a,b,c,d) \
OP_INTER_4 (OP_2, a,b,c,d) \
OP_INTER_4 (OP_3, a,b,c,d) \
OP_INTER_4 (OP_4, a,b,c,d) \
OP_INTER_4 (OP_5, a,b,c,d) \
OP_INTER_4 (OP_6, a,b,c,d) \
OP_INTER_4 (OP_7, a,b,c,d) \
OP_INTER_4 (OP_1, a,b,c,d) \
OP_INTER_4 (OP_2, a,b,c,d) \
OP_INTER_4 (OP_8, a,b,c,d) \
OP_INTER_4 (OP_4, a,b,c,d) \
OP_INTER_4 (OP_5, a,b,c,d) \
OP_INTER_4 (OP_9, a,b,c,d) \
#define V4R \
{ \
V4G (0, 1, 2, 3) \
V4G (4, 5, 6, 7) \
}
#endif
#define V4_ROUND(r) { GET_SIGMA_PTR_128(r); V4R }
#define V4_LOAD_MSG_1(w, m, i) \
{ \
__m128i m0, m1, m2, m3; \
__m128i t0, t1, t2, t3; \
m0 = LOADU_128((m) + ((i) + 0 * 4) * 16); \
m1 = LOADU_128((m) + ((i) + 1 * 4) * 16); \
m2 = LOADU_128((m) + ((i) + 2 * 4) * 16); \
m3 = LOADU_128((m) + ((i) + 3 * 4) * 16); \
t0 = _mm_unpacklo_epi32(m0, m1); \
t1 = _mm_unpackhi_epi32(m0, m1); \
t2 = _mm_unpacklo_epi32(m2, m3); \
t3 = _mm_unpackhi_epi32(m2, m3); \
w[(i) * 4 + 0] = _mm_unpacklo_epi64(t0, t2); \
w[(i) * 4 + 1] = _mm_unpackhi_epi64(t0, t2); \
w[(i) * 4 + 2] = _mm_unpacklo_epi64(t1, t3); \
w[(i) * 4 + 3] = _mm_unpackhi_epi64(t1, t3); \
}
#define V4_LOAD_MSG(w, m) \
{ \
V4_LOAD_MSG_1 (w, m, 0) \
V4_LOAD_MSG_1 (w, m, 1) \
V4_LOAD_MSG_1 (w, m, 2) \
V4_LOAD_MSG_1 (w, m, 3) \
}
#define V4_LOAD_UNPACK_PAIR_128(src32, i, d0, d1) \
{ \
const __m128i v0 = LOAD_128_FROM_STRUCT((src32) + (i ) * 4); \
const __m128i v1 = LOAD_128_FROM_STRUCT((src32) + (i + 1) * 4); \
d0 = _mm_unpacklo_epi32(v0, v1); \
d1 = _mm_unpackhi_epi32(v0, v1); \
}
#define V4_UNPACK_PAIR_128(dest32, i, s0, s1) \
{ \
STORE_128_TO_STRUCT((dest32) + i * 4 , _mm_unpacklo_epi64(s0, s1)); \
STORE_128_TO_STRUCT((dest32) + i * 4 + 16, _mm_unpackhi_epi64(s0, s1)); \
}
#define V4_UNPACK_STATE(dest32, src32) \
{ \
__m128i t0, t1, t2, t3, t4, t5, t6, t7; \
V4_LOAD_UNPACK_PAIR_128(src32, 0, t0, t1) \
V4_LOAD_UNPACK_PAIR_128(src32, 2, t2, t3) \
V4_LOAD_UNPACK_PAIR_128(src32, 4, t4, t5) \
V4_LOAD_UNPACK_PAIR_128(src32, 6, t6, t7) \
V4_UNPACK_PAIR_128(dest32, 0, t0, t2) \
V4_UNPACK_PAIR_128(dest32, 8, t1, t3) \
V4_UNPACK_PAIR_128(dest32, 1, t4, t6) \
V4_UNPACK_PAIR_128(dest32, 9, t5, t7) \
}
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_128BIT
BLAKE2S_ATTRIB_128BIT
#endif
void
Z7_FASTCALL
Blake2sp_Compress2_V128_Fast(UInt32 *s_items, const Byte *data, const Byte *end)
{
// PrintStates2(s_items, 8, 16);
size_t pos = 0;
pos /= 2;
do
{
#if defined(Z7_BLAKE2S_USE_SSSE3) && \
!defined(Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED)
const __m128i r8 = k_r8;
const __m128i r16 = k_r16;
#endif
__m128i w[16];
__m128i v[16];
UInt32 *s;
V4_LOAD_MSG(w, data)
s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
{
__m128i ctr = LOAD_128_FROM_STRUCT(s + 64);
D_ADD_EPI64_128 (ctr, k_inc);
STORE_128_TO_STRUCT(s + 64, ctr);
v[12] = XOR_128 (GET_128_IV_WAY4(4), _mm_shuffle_epi32(ctr, _MM_SHUFFLE(0, 0, 0, 0)));
v[13] = XOR_128 (GET_128_IV_WAY4(5), _mm_shuffle_epi32(ctr, _MM_SHUFFLE(1, 1, 1, 1)));
}
v[ 8] = GET_128_IV_WAY4(0);
v[ 9] = GET_128_IV_WAY4(1);
v[10] = GET_128_IV_WAY4(2);
v[11] = GET_128_IV_WAY4(3);
v[14] = GET_128_IV_WAY4(6);
v[15] = GET_128_IV_WAY4(7);
#define LOAD_STATE_128_FROM_STRUCT(i) \
v[i] = LOAD_128_FROM_STRUCT(s + (i) * 4);
#define UPDATE_STATE_128_IN_STRUCT(i) \
STORE_128_TO_STRUCT(s + (i) * 4, XOR_128( \
XOR_128(v[i], v[(i) + 8]), \
LOAD_128_FROM_STRUCT(s + (i) * 4)));
REP8_MACRO (LOAD_STATE_128_FROM_STRUCT)
ROUNDS_LOOP (V4_ROUND)
REP8_MACRO (UPDATE_STATE_128_IN_STRUCT)
data += Z7_BLAKE2S_BLOCK_SIZE * 4;
pos += Z7_BLAKE2S_BLOCK_SIZE * 4 / 2;
pos &= SUPER_BLOCK_SIZE / 2 - 1;
}
while (data != end);
}
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_128BIT
BLAKE2S_ATTRIB_128BIT
#endif
void
Z7_FASTCALL
Blake2sp_Final_V128_Fast(UInt32 *states)
{
const __m128i ctr = LOAD_128_FROM_STRUCT(states + 64);
// printf("\nBlake2sp_Compress2_V128_Fast_Final4\n");
// PrintStates2(states, 8, 16);
{
ptrdiff_t pos = 8 * 4;
do
{
UInt32 *src32 = states + (size_t)(pos * 1);
UInt32 *dest32 = states + (size_t)(pos * 2);
V4_UNPACK_STATE(dest32, src32)
pos -= 8 * 4;
}
while (pos >= 0);
}
{
unsigned k;
for (k = 0; k < 8; k++)
{
UInt32 *s = states + (size_t)k * 16;
STORE_128_TO_STRUCT (STATE_T(s), ctr);
}
}
// PrintStates2(states, 8, 16);
}
#ifdef Z7_BLAKE2S_USE_AVX2
#define ADD_256(a, b) _mm256_add_epi32(a, b)
#define XOR_256(a, b) _mm256_xor_si256(a, b)
#if 1 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS)
#define MM256_ROR_EPI32 _mm256_ror_epi32
#define Z7_MM256_ROR_EPI32_IS_SUPPORTED
#ifdef Z7_BLAKE2S_USE_AVX2_WAY2
#define LOAD_ROTATE_CONSTS_256
#endif
#else
#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#ifdef Z7_BLAKE2S_USE_AVX2_WAY2
#define LOAD_ROTATE_CONSTS_256 \
const __m256i r8 = k_r8_256; \
const __m256i r16 = k_r16_256;
#endif // AVX2_WAY2
#define MM256_ROR_EPI32(r, c) ( \
( 8==(c)) ? _mm256_shuffle_epi8(r,r8) \
: (16==(c)) ? _mm256_shuffle_epi8(r,r16) \
: _mm256_or_si256( \
_mm256_srli_epi32((r), (c)), \
_mm256_slli_epi32((r), 32-(c))))
#endif // WAY_SLOW
#endif
#define D_ADD_256(dest, src) dest = ADD_256(dest, src)
#define D_XOR_256(dest, src) dest = XOR_256(dest, src)
#define LOADU_256(p) _mm256_loadu_si256((const __m256i *)(const void *)(p))
#ifdef Z7_BLAKE2S_USE_AVX2_FAST
#ifdef Z7_MM256_ROR_EPI32_IS_SUPPORTED
#define ROT_256_16(x) MM256_ROR_EPI32((x), 16)
#define ROT_256_12(x) MM256_ROR_EPI32((x), 12)
#define ROT_256_8(x) MM256_ROR_EPI32((x), 8)
#define ROT_256_7(x) MM256_ROR_EPI32((x), 7)
#else
#define ROTATE8 _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, \
12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)
#define ROTATE16 _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, \
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)
#define ROT_256_16(x) _mm256_shuffle_epi8((x), ROTATE16)
#define ROT_256_12(x) _mm256_or_si256(_mm256_srli_epi32((x), 12), _mm256_slli_epi32((x), 20))
#define ROT_256_8(x) _mm256_shuffle_epi8((x), ROTATE8)
#define ROT_256_7(x) _mm256_or_si256(_mm256_srli_epi32((x), 7), _mm256_slli_epi32((x), 25))
#endif
#define D_ROT_256_7(dest) dest = ROT_256_7(dest)
#define D_ROT_256_8(dest) dest = ROT_256_8(dest)
#define D_ROT_256_12(dest) dest = ROT_256_12(dest)
#define D_ROT_256_16(dest) dest = ROT_256_16(dest)
#define LOAD_256(p) _mm256_load_si256((const __m256i *)(const void *)(p))
#ifdef Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED
#define STOREU_256(p, r) _mm256_storeu_si256((__m256i *)(void *)(p), r)
#define LOAD_256_FROM_STRUCT(p) LOADU_256(p)
#define STORE_256_TO_STRUCT(p, r) STOREU_256(p, r)
#else
// if struct is aligned for 32-bytes
#define STORE_256(p, r) _mm256_store_si256((__m256i *)(void *)(p), r)
#define LOAD_256_FROM_STRUCT(p) LOAD_256(p)
#define STORE_256_TO_STRUCT(p, r) STORE_256(p, r)
#endif
#endif // Z7_BLAKE2S_USE_AVX2_FAST
#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#if 0
#define DIAG_PERM2(s) \
{ \
const __m256i a = LOAD_256_FROM_STRUCT((s) ); \
const __m256i b = LOAD_256_FROM_STRUCT((s) + NSW); \
STORE_256_TO_STRUCT((s ), _mm256_permute2x128_si256(a, b, 0x20)); \
STORE_256_TO_STRUCT((s + NSW), _mm256_permute2x128_si256(a, b, 0x31)); \
}
#else
#define DIAG_PERM2(s) \
{ \
const __m128i a = LOAD_128_FROM_STRUCT((s) + 4); \
const __m128i b = LOAD_128_FROM_STRUCT((s) + NSW); \
STORE_128_TO_STRUCT((s) + NSW, a); \
STORE_128_TO_STRUCT((s) + 4 , b); \
}
#endif
#define DIAG_PERM8(s_items) \
{ \
DIAG_PERM2(s_items) \
DIAG_PERM2(s_items + NSW * 2) \
DIAG_PERM2(s_items + NSW * 4) \
DIAG_PERM2(s_items + NSW * 6) \
}
#define AXR256(a, b, d, shift) \
D_ADD_256(a, b); \
D_XOR_256(d, a); \
d = MM256_ROR_EPI32(d, shift); \
#ifdef Z7_BLAKE2S_USE_GATHER
#define TABLE_GATHER_256_4(a0,a1,a2,a3) \
a0,a1,a2,a3, a0+16,a1+16,a2+16,a3+16
#define TABLE_GATHER_256( \
a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
{ TABLE_GATHER_256_4(a0,a2,a4,a6), \
TABLE_GATHER_256_4(a1,a3,a5,a7), \
TABLE_GATHER_256_4(a8,a10,a12,a14), \
TABLE_GATHER_256_4(a9,a11,a13,a15) }
MY_ALIGN(64)
static const UInt32 k_Blake2s_Sigma_gather256[BLAKE2S_NUM_ROUNDS][16 * 2] =
{ SIGMA_TABLE(TABLE_GATHER_256) };
#define GET_SIGMA(r) \
const UInt32 * const sigma = k_Blake2s_Sigma_gather256[r];
#define AXR2_LOAD_INDEXES_AVX(sigma_index) \
const __m256i i01234567 = LOAD_256(sigma + (sigma_index));
#define SET_ROW_FROM_SIGMA_AVX(in) \
_mm256_i32gather_epi32((const void *)(in), i01234567, 4)
#define SIGMA_INTERLEAVE 8
#define SIGMA_HALF_ROW_SIZE 16
#else // !Z7_BLAKE2S_USE_GATHER
#define GET_SIGMA(r) \
const Byte * const sigma = k_Blake2s_Sigma_4[r];
#define AXR2_LOAD_INDEXES_AVX(sigma_index) \
AXR2_LOAD_INDEXES(sigma_index)
#define SET_ROW_FROM_SIGMA_AVX(in) \
MY_mm256_set_m128i( \
SET_ROW_FROM_SIGMA_W((in) + Z7_BLAKE2S_BLOCK_SIZE), \
SET_ROW_FROM_SIGMA_W(in))
#define SIGMA_INTERLEAVE 1
#define SIGMA_HALF_ROW_SIZE 8
#endif // !Z7_BLAKE2S_USE_GATHER
#define ROTATE_WORDS_TO_RIGHT_256(a, n) \
a = _mm256_shuffle_epi32(a, _MM_SHUFFLE((3+n)&3, (2+n)&3, (1+n)&3, (0+n)&3));
#ifdef Z7_BLAKE2S_USE_AVX2_WAY2
#define AXR2_A(sigma_index, shift1, shift2) \
AXR2_LOAD_INDEXES_AVX(sigma_index) \
D_ADD_256( a0, SET_ROW_FROM_SIGMA_AVX(data)); \
AXR256(a0, b0, d0, shift1) \
AXR256(c0, d0, b0, shift2) \
#define AXR4_A(sigma_index) \
{ AXR2_A(sigma_index, 16, 12) } \
{ AXR2_A(sigma_index + SIGMA_INTERLEAVE, 8, 7) }
#define EE1(r) \
{ GET_SIGMA(r) \
AXR4_A(0) \
ROTATE_WORDS_TO_RIGHT_256(b0, 1) \
ROTATE_WORDS_TO_RIGHT_256(c0, 2) \
ROTATE_WORDS_TO_RIGHT_256(d0, 3) \
AXR4_A(SIGMA_HALF_ROW_SIZE) \
ROTATE_WORDS_TO_RIGHT_256(b0, 3) \
ROTATE_WORDS_TO_RIGHT_256(c0, 2) \
ROTATE_WORDS_TO_RIGHT_256(d0, 1) \
}
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_AVX2
BLAKE2S_ATTRIB_AVX2
#endif
void
Z7_FASTCALL
Blake2sp_Compress2_AVX2_Way2(UInt32 *s_items, const Byte *data, const Byte *end)
{
size_t pos = 0;
end -= Z7_BLAKE2S_BLOCK_SIZE;
if (data != end)
{
LOAD_ROTATE_CONSTS_256
DIAG_PERM8(s_items)
do
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
__m256i a0, b0, c0, d0;
{
const __m128i inc = k_inc;
__m128i d0_128 = LOAD_128_FROM_STRUCT (STATE_T(s));
__m128i d1_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW));
D_ADD_EPI64_128(d0_128, inc);
D_ADD_EPI64_128(d1_128, inc);
STORE_128_TO_STRUCT (STATE_T(s ), d0_128);
STORE_128_TO_STRUCT (STATE_T(s + NSW), d1_128);
d0 = MY_mm256_set_m128i(d1_128, d0_128);
D_XOR_256(d0, k_iv4_256);
}
c0 = SET_FROM_128(k_iv0_128);
a0 = LOAD_256_FROM_STRUCT(s + NSW * 0);
b0 = LOAD_256_FROM_STRUCT(s + NSW * 1);
ROUNDS_LOOP (EE1)
D_XOR_256(a0, c0);
D_XOR_256(b0, d0);
D_XOR_256(a0, LOAD_256_FROM_STRUCT(s + NSW * 0));
D_XOR_256(b0, LOAD_256_FROM_STRUCT(s + NSW * 1));
STORE_256_TO_STRUCT(s + NSW * 0, a0);
STORE_256_TO_STRUCT(s + NSW * 1, b0);
data += Z7_BLAKE2S_BLOCK_SIZE * 2;
pos += Z7_BLAKE2S_BLOCK_SIZE * 2;
pos &= SUPER_BLOCK_MASK;
}
while (data < end);
DIAG_PERM8(s_items)
if (data != end)
return;
}
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
Z7_BLAKE2S_CompressSingleBlock(s, data);
}
}
#endif // Z7_BLAKE2S_USE_AVX2_WAY2
#ifdef Z7_BLAKE2S_USE_AVX2_WAY4
#define AXR2_X(sigma_index, shift1, shift2) \
AXR2_LOAD_INDEXES_AVX(sigma_index) \
D_ADD_256( a0, SET_ROW_FROM_SIGMA_AVX(data)); \
D_ADD_256( a1, SET_ROW_FROM_SIGMA_AVX((data) + Z7_BLAKE2S_BLOCK_SIZE * 2)); \
AXR256(a0, b0, d0, shift1) \
AXR256(a1, b1, d1, shift1) \
AXR256(c0, d0, b0, shift2) \
AXR256(c1, d1, b1, shift2) \
#define AXR4_X(sigma_index) \
{ AXR2_X(sigma_index, 16, 12) } \
{ AXR2_X(sigma_index + SIGMA_INTERLEAVE, 8, 7) }
#define EE2(r) \
{ GET_SIGMA(r) \
AXR4_X(0) \
ROTATE_WORDS_TO_RIGHT_256(b0, 1) \
ROTATE_WORDS_TO_RIGHT_256(b1, 1) \
ROTATE_WORDS_TO_RIGHT_256(c0, 2) \
ROTATE_WORDS_TO_RIGHT_256(c1, 2) \
ROTATE_WORDS_TO_RIGHT_256(d0, 3) \
ROTATE_WORDS_TO_RIGHT_256(d1, 3) \
AXR4_X(SIGMA_HALF_ROW_SIZE) \
ROTATE_WORDS_TO_RIGHT_256(b0, 3) \
ROTATE_WORDS_TO_RIGHT_256(b1, 3) \
ROTATE_WORDS_TO_RIGHT_256(c0, 2) \
ROTATE_WORDS_TO_RIGHT_256(c1, 2) \
ROTATE_WORDS_TO_RIGHT_256(d0, 1) \
ROTATE_WORDS_TO_RIGHT_256(d1, 1) \
}
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_AVX2
BLAKE2S_ATTRIB_AVX2
#endif
void
Z7_FASTCALL
Blake2sp_Compress2_AVX2_Way4(UInt32 *s_items, const Byte *data, const Byte *end)
{
size_t pos = 0;
if ((size_t)(end - data) >= Z7_BLAKE2S_BLOCK_SIZE * 4)
{
#ifndef Z7_MM256_ROR_EPI32_IS_SUPPORTED
const __m256i r8 = k_r8_256;
const __m256i r16 = k_r16_256;
#endif
end -= Z7_BLAKE2S_BLOCK_SIZE * 3;
DIAG_PERM8(s_items)
do
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
__m256i a0, b0, c0, d0;
__m256i a1, b1, c1, d1;
{
const __m128i inc = k_inc;
__m128i d0_128 = LOAD_128_FROM_STRUCT (STATE_T(s));
__m128i d1_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW));
__m128i d2_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW * 2));
__m128i d3_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW * 3));
D_ADD_EPI64_128(d0_128, inc);
D_ADD_EPI64_128(d1_128, inc);
D_ADD_EPI64_128(d2_128, inc);
D_ADD_EPI64_128(d3_128, inc);
STORE_128_TO_STRUCT (STATE_T(s ), d0_128);
STORE_128_TO_STRUCT (STATE_T(s + NSW * 1), d1_128);
STORE_128_TO_STRUCT (STATE_T(s + NSW * 2), d2_128);
STORE_128_TO_STRUCT (STATE_T(s + NSW * 3), d3_128);
d0 = MY_mm256_set_m128i(d1_128, d0_128);
d1 = MY_mm256_set_m128i(d3_128, d2_128);
D_XOR_256(d0, k_iv4_256);
D_XOR_256(d1, k_iv4_256);
}
c1 = c0 = SET_FROM_128(k_iv0_128);
a0 = LOAD_256_FROM_STRUCT(s + NSW * 0);
b0 = LOAD_256_FROM_STRUCT(s + NSW * 1);
a1 = LOAD_256_FROM_STRUCT(s + NSW * 2);
b1 = LOAD_256_FROM_STRUCT(s + NSW * 3);
ROUNDS_LOOP (EE2)
D_XOR_256(a0, c0);
D_XOR_256(b0, d0);
D_XOR_256(a1, c1);
D_XOR_256(b1, d1);
D_XOR_256(a0, LOAD_256_FROM_STRUCT(s + NSW * 0));
D_XOR_256(b0, LOAD_256_FROM_STRUCT(s + NSW * 1));
D_XOR_256(a1, LOAD_256_FROM_STRUCT(s + NSW * 2));
D_XOR_256(b1, LOAD_256_FROM_STRUCT(s + NSW * 3));
STORE_256_TO_STRUCT(s + NSW * 0, a0);
STORE_256_TO_STRUCT(s + NSW * 1, b0);
STORE_256_TO_STRUCT(s + NSW * 2, a1);
STORE_256_TO_STRUCT(s + NSW * 3, b1);
data += Z7_BLAKE2S_BLOCK_SIZE * 4;
pos += Z7_BLAKE2S_BLOCK_SIZE * 4;
pos &= SUPER_BLOCK_MASK;
}
while (data < end);
DIAG_PERM8(s_items)
end += Z7_BLAKE2S_BLOCK_SIZE * 3;
}
if (data == end)
return;
// Z7_BLAKE2S_Compress2_V128(s_items, data, end, pos);
do
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
Z7_BLAKE2S_CompressSingleBlock(s, data);
data += Z7_BLAKE2S_BLOCK_SIZE;
pos += Z7_BLAKE2S_BLOCK_SIZE;
pos &= SUPER_BLOCK_MASK;
}
while (data != end);
}
#endif // Z7_BLAKE2S_USE_AVX2_WAY4
#endif // Z7_BLAKE2S_USE_AVX2_WAY_SLOW
// ---------------------------------------------------------
#ifdef Z7_BLAKE2S_USE_AVX2_FAST
#define OP256_L(a, i) D_ADD_256 (V(a, 0), \
LOAD_256((const Byte *)(w) + GET_SIGMA_VAL_256(2*(a)+(i))));
#define OP256_0(a) OP256_L(a, 0)
#define OP256_7(a) OP256_L(a, 1)
#define OP256_1(a) D_ADD_256 (V(a, 0), V(a, 1));
#define OP256_2(a) D_XOR_256 (V(a, 3), V(a, 0));
#define OP256_4(a) D_ADD_256 (V(a, 2), V(a, 3));
#define OP256_5(a) D_XOR_256 (V(a, 1), V(a, 2));
#define OP256_3(a) D_ROT_256_16 (V(a, 3));
#define OP256_6(a) D_ROT_256_12 (V(a, 1));
#define OP256_8(a) D_ROT_256_8 (V(a, 3));
#define OP256_9(a) D_ROT_256_7 (V(a, 1));
#if 0 || 1 && defined(MY_CPU_X86)
#define V8_G(a) \
OP256_0 (a) \
OP256_1 (a) \
OP256_2 (a) \
OP256_3 (a) \
OP256_4 (a) \
OP256_5 (a) \
OP256_6 (a) \
OP256_7 (a) \
OP256_1 (a) \
OP256_2 (a) \
OP256_8 (a) \
OP256_4 (a) \
OP256_5 (a) \
OP256_9 (a) \
#define V8R { \
V8_G (0); \
V8_G (1); \
V8_G (2); \
V8_G (3); \
V8_G (4); \
V8_G (5); \
V8_G (6); \
V8_G (7); \
}
#else
#define OP256_INTER_4(op, a,b,c,d) \
op (a) \
op (b) \
op (c) \
op (d) \
#define V8_G(a,b,c,d) \
OP256_INTER_4 (OP256_0, a,b,c,d) \
OP256_INTER_4 (OP256_1, a,b,c,d) \
OP256_INTER_4 (OP256_2, a,b,c,d) \
OP256_INTER_4 (OP256_3, a,b,c,d) \
OP256_INTER_4 (OP256_4, a,b,c,d) \
OP256_INTER_4 (OP256_5, a,b,c,d) \
OP256_INTER_4 (OP256_6, a,b,c,d) \
OP256_INTER_4 (OP256_7, a,b,c,d) \
OP256_INTER_4 (OP256_1, a,b,c,d) \
OP256_INTER_4 (OP256_2, a,b,c,d) \
OP256_INTER_4 (OP256_8, a,b,c,d) \
OP256_INTER_4 (OP256_4, a,b,c,d) \
OP256_INTER_4 (OP256_5, a,b,c,d) \
OP256_INTER_4 (OP256_9, a,b,c,d) \
#define V8R { \
V8_G (0, 1, 2, 3) \
V8_G (4, 5, 6, 7) \
}
#endif
#define V8_ROUND(r) { GET_SIGMA_PTR_256(r); V8R }
// for debug:
// #define Z7_BLAKE2S_PERMUTE_WITH_GATHER
#if defined(Z7_BLAKE2S_PERMUTE_WITH_GATHER)
// gather instruction is slow.
#define V8_LOAD_MSG(w, m) \
{ \
unsigned i; \
for (i = 0; i < 16; ++i) { \
w[i] = _mm256_i32gather_epi32( \
(const void *)((m) + i * sizeof(UInt32)),\
_mm256_set_epi32(0x70, 0x60, 0x50, 0x40, 0x30, 0x20, 0x10, 0x00), \
sizeof(UInt32)); \
} \
}
#else // !Z7_BLAKE2S_PERMUTE_WITH_GATHER
#define V8_LOAD_MSG_2(w, a0, a1) \
{ \
(w)[0] = _mm256_permute2x128_si256(a0, a1, 0x20); \
(w)[4] = _mm256_permute2x128_si256(a0, a1, 0x31); \
}
#define V8_LOAD_MSG_4(w, z0, z1, z2, z3) \
{ \
__m256i s0, s1, s2, s3; \
s0 = _mm256_unpacklo_epi64(z0, z1); \
s1 = _mm256_unpackhi_epi64(z0, z1); \
s2 = _mm256_unpacklo_epi64(z2, z3); \
s3 = _mm256_unpackhi_epi64(z2, z3); \
V8_LOAD_MSG_2((w) + 0, s0, s2) \
V8_LOAD_MSG_2((w) + 1, s1, s3) \
}
#define V8_LOAD_MSG_0(t0, t1, m) \
{ \
__m256i m0, m1; \
m0 = LOADU_256(m); \
m1 = LOADU_256((m) + 2 * 32); \
t0 = _mm256_unpacklo_epi32(m0, m1); \
t1 = _mm256_unpackhi_epi32(m0, m1); \
}
#define V8_LOAD_MSG_8(w, m) \
{ \
__m256i t0, t1, t2, t3, t4, t5, t6, t7; \
V8_LOAD_MSG_0(t0, t4, (m) + 0 * 4 * 32) \
V8_LOAD_MSG_0(t1, t5, (m) + 1 * 4 * 32) \
V8_LOAD_MSG_0(t2, t6, (m) + 2 * 4 * 32) \
V8_LOAD_MSG_0(t3, t7, (m) + 3 * 4 * 32) \
V8_LOAD_MSG_4((w) , t0, t1, t2, t3) \
V8_LOAD_MSG_4((w) + 2, t4, t5, t6, t7) \
}
#define V8_LOAD_MSG(w, m) \
{ \
V8_LOAD_MSG_8(w, m) \
V8_LOAD_MSG_8((w) + 8, (m) + 32) \
}
#endif // !Z7_BLAKE2S_PERMUTE_WITH_GATHER
#define V8_PERM_PAIR_STORE(u, a0, a2) \
{ \
STORE_256_TO_STRUCT((u), _mm256_permute2x128_si256(a0, a2, 0x20)); \
STORE_256_TO_STRUCT((u) + 8, _mm256_permute2x128_si256(a0, a2, 0x31)); \
}
#define V8_UNPACK_STORE_4(u, z0, z1, z2, z3) \
{ \
__m256i s0, s1, s2, s3; \
s0 = _mm256_unpacklo_epi64(z0, z1); \
s1 = _mm256_unpackhi_epi64(z0, z1); \
s2 = _mm256_unpacklo_epi64(z2, z3); \
s3 = _mm256_unpackhi_epi64(z2, z3); \
V8_PERM_PAIR_STORE(u + 0, s0, s2) \
V8_PERM_PAIR_STORE(u + 2, s1, s3) \
}
#define V8_UNPACK_STORE_0(src32, d0, d1) \
{ \
const __m256i v0 = LOAD_256_FROM_STRUCT ((src32) ); \
const __m256i v1 = LOAD_256_FROM_STRUCT ((src32) + 8); \
d0 = _mm256_unpacklo_epi32(v0, v1); \
d1 = _mm256_unpackhi_epi32(v0, v1); \
}
#define V8_UNPACK_STATE(dest32, src32) \
{ \
__m256i t0, t1, t2, t3, t4, t5, t6, t7; \
V8_UNPACK_STORE_0 ((src32) + 16 * 0, t0, t4) \
V8_UNPACK_STORE_0 ((src32) + 16 * 1, t1, t5) \
V8_UNPACK_STORE_0 ((src32) + 16 * 2, t2, t6) \
V8_UNPACK_STORE_0 ((src32) + 16 * 3, t3, t7) \
V8_UNPACK_STORE_4 ((__m256i *)(void *)(dest32) , t0, t1, t2, t3) \
V8_UNPACK_STORE_4 ((__m256i *)(void *)(dest32) + 4, t4, t5, t6, t7) \
}
#define V8_LOAD_STATE_256_FROM_STRUCT(i) \
v[i] = LOAD_256_FROM_STRUCT(s_items + (i) * 8);
#if 0 || 0 && defined(MY_CPU_X86)
#define Z7_BLAKE2S_AVX2_FAST_USE_STRUCT
#endif
#ifdef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT
// this branch doesn't use (iv) array
// so register pressure can be lower.
// it can be faster sometimes
#define V8_LOAD_STATE_256(i) V8_LOAD_STATE_256_FROM_STRUCT(i)
#define V8_UPDATE_STATE_256(i) \
{ \
STORE_256_TO_STRUCT(s_items + (i) * 8, XOR_256( \
XOR_256(v[i], v[(i) + 8]), \
LOAD_256_FROM_STRUCT(s_items + (i) * 8))); \
}
#else
// it uses more variables (iv) registers
// it's better for gcc
// maybe that branch is better, if register pressure will be lower (avx512)
#define V8_LOAD_STATE_256(i) { iv[i] = v[i]; }
#define V8_UPDATE_STATE_256(i) { v[i] = XOR_256(XOR_256(v[i], v[i + 8]), iv[i]); }
#define V8_STORE_STATE_256(i) { STORE_256_TO_STRUCT(s_items + (i) * 8, v[i]); }
#endif
#if 0
// use loading constants from memory
#define KK8(n) KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n)
MY_ALIGN(64)
static const UInt32 k_Blake2s_IV_WAY8[]=
{
KK8(0), KK8(1), KK8(2), KK8(3), KK8(4), KK8(5), KK8(6), KK8(7)
};
#define GET_256_IV_WAY8(i) LOAD_256(k_Blake2s_IV_WAY8 + 8 * (i))
#else
// use constant generation:
#define GET_256_IV_WAY8(i) _mm256_set1_epi32((Int32)KIV(i))
#endif
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_AVX2
BLAKE2S_ATTRIB_AVX2
#endif
void
Z7_FASTCALL
Blake2sp_Compress2_AVX2_Fast(UInt32 *s_items, const Byte *data, const Byte *end)
{
#ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT
__m256i v[16];
#endif
// PrintStates2(s_items, 8, 16);
#ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT
REP8_MACRO (V8_LOAD_STATE_256_FROM_STRUCT)
#endif
do
{
__m256i w[16];
#ifdef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT
__m256i v[16];
#else
__m256i iv[8];
#endif
V8_LOAD_MSG(w, data)
{
// we use load/store ctr inside loop to reduce register pressure:
#if 1 || 1 && defined(MY_CPU_X86)
const __m256i ctr = _mm256_add_epi64(
LOAD_256_FROM_STRUCT(s_items + 64),
_mm256_set_epi32(
0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE,
0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE));
STORE_256_TO_STRUCT(s_items + 64, ctr);
#else
const UInt64 ctr64 = *(const UInt64 *)(const void *)(s_items + 64)
+ Z7_BLAKE2S_BLOCK_SIZE;
const __m256i ctr = _mm256_set_epi64x(0, (Int64)ctr64, 0, (Int64)ctr64);
*(UInt64 *)(void *)(s_items + 64) = ctr64;
#endif
v[12] = XOR_256 (GET_256_IV_WAY8(4), _mm256_shuffle_epi32(ctr, _MM_SHUFFLE(0, 0, 0, 0)));
v[13] = XOR_256 (GET_256_IV_WAY8(5), _mm256_shuffle_epi32(ctr, _MM_SHUFFLE(1, 1, 1, 1)));
}
v[ 8] = GET_256_IV_WAY8(0);
v[ 9] = GET_256_IV_WAY8(1);
v[10] = GET_256_IV_WAY8(2);
v[11] = GET_256_IV_WAY8(3);
v[14] = GET_256_IV_WAY8(6);
v[15] = GET_256_IV_WAY8(7);
REP8_MACRO (V8_LOAD_STATE_256)
ROUNDS_LOOP (V8_ROUND)
REP8_MACRO (V8_UPDATE_STATE_256)
data += SUPER_BLOCK_SIZE;
}
while (data != end);
#ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT
REP8_MACRO (V8_STORE_STATE_256)
#endif
}
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_AVX2
BLAKE2S_ATTRIB_AVX2
#endif
void
Z7_FASTCALL
Blake2sp_Final_AVX2_Fast(UInt32 *states)
{
const __m128i ctr = LOAD_128_FROM_STRUCT(states + 64);
// PrintStates2(states, 8, 16);
V8_UNPACK_STATE(states, states)
// PrintStates2(states, 8, 16);
{
unsigned k;
for (k = 0; k < 8; k++)
{
UInt32 *s = states + (size_t)k * 16;
STORE_128_TO_STRUCT (STATE_T(s), ctr);
}
}
// PrintStates2(states, 8, 16);
// printf("\nafter V8_UNPACK_STATE \n");
}
#endif // Z7_BLAKE2S_USE_AVX2_FAST
#endif // avx2
#endif // vector
/*
#define Blake2s_Increment_Counter(s, inc) \
{ STATE_T(s)[0] += (inc); STATE_T(s)[1] += (STATE_T(s)[0] < (inc)); }
#define Blake2s_Increment_Counter_Small(s, inc) \
{ STATE_T(s)[0] += (inc); }
*/
#define Blake2s_Set_LastBlock(s) \
{ STATE_F(s)[0] = BLAKE2S_FINAL_FLAG; /* STATE_F(s)[1] = p->u.header.lastNode_f1; */ }
#if 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL >= 1600
// good for vs2022
#define LOOP_8(mac) { unsigned kkk; for (kkk = 0; kkk < 8; kkk++) mac(kkk) }
#else
// good for Z7_BLAKE2S_UNROLL for GCC9 (arm*/x86*) and MSC_VER_1400-x64.
#define LOOP_8(mac) { REP8_MACRO(mac) }
#endif
static
Z7_FORCE_INLINE
// Z7_NO_INLINE
void
Z7_FASTCALL
Blake2s_Compress(UInt32 *s, const Byte *input)
{
UInt32 m[16];
UInt32 v[16];
{
unsigned i;
for (i = 0; i < 16; i++)
m[i] = GetUi32(input + i * 4);
}
#define INIT_v_FROM_s(i) v[i] = s[i];
LOOP_8(INIT_v_FROM_s)
// Blake2s_Increment_Counter(s, Z7_BLAKE2S_BLOCK_SIZE)
{
const UInt32 t0 = STATE_T(s)[0] + Z7_BLAKE2S_BLOCK_SIZE;
const UInt32 t1 = STATE_T(s)[1] + (t0 < Z7_BLAKE2S_BLOCK_SIZE);
STATE_T(s)[0] = t0;
STATE_T(s)[1] = t1;
v[12] = t0 ^ KIV(4);
v[13] = t1 ^ KIV(5);
}
// v[12] = STATE_T(s)[0] ^ KIV(4);
// v[13] = STATE_T(s)[1] ^ KIV(5);
v[14] = STATE_F(s)[0] ^ KIV(6);
v[15] = STATE_F(s)[1] ^ KIV(7);
v[ 8] = KIV(0);
v[ 9] = KIV(1);
v[10] = KIV(2);
v[11] = KIV(3);
// PrintStates2((const UInt32 *)v, 1, 16);
#define ADD_SIGMA(a, index) V(a, 0) += *(const UInt32 *)GET_SIGMA_PTR(m, sigma[index]);
#define ADD32M(dest, src, a) V(a, dest) += V(a, src);
#define XOR32M(dest, src, a) V(a, dest) ^= V(a, src);
#define RTR32M(dest, shift, a) V(a, dest) = rotrFixed(V(a, dest), shift);
// big interleaving can provides big performance gain, if scheduler queues are small.
#if 0 || 1 && defined(MY_CPU_X86)
// interleave-1: for small register number (x86-32bit)
#define G2(index, a, x, y) \
ADD_SIGMA (a, (index) + 2 * 0) \
ADD32M (0, 1, a) \
XOR32M (3, 0, a) \
RTR32M (3, x, a) \
ADD32M (2, 3, a) \
XOR32M (1, 2, a) \
RTR32M (1, y, a) \
#define G(a) \
G2(a * 2 , a, 16, 12) \
G2(a * 2 + 1, a, 8, 7) \
#define R2 \
G(0) \
G(1) \
G(2) \
G(3) \
G(4) \
G(5) \
G(6) \
G(7) \
#elif 0 || 1 && defined(MY_CPU_X86_OR_AMD64)
// interleave-2: is good if the number of registers is not big (x86-64).
#define REP2(mac, dest, src, a, b) \
mac(dest, src, a) \
mac(dest, src, b)
#define G2(index, a, b, x, y) \
ADD_SIGMA (a, (index) + 2 * 0) \
ADD_SIGMA (b, (index) + 2 * 1) \
REP2 (ADD32M, 0, 1, a, b) \
REP2 (XOR32M, 3, 0, a, b) \
REP2 (RTR32M, 3, x, a, b) \
REP2 (ADD32M, 2, 3, a, b) \
REP2 (XOR32M, 1, 2, a, b) \
REP2 (RTR32M, 1, y, a, b) \
#define G(a, b) \
G2(a * 2 , a, b, 16, 12) \
G2(a * 2 + 1, a, b, 8, 7) \
#define R2 \
G(0, 1) \
G(2, 3) \
G(4, 5) \
G(6, 7) \
#else
// interleave-4:
// it has big register pressure for x86/x64.
// and MSVC compilers for x86/x64 are slow for this branch.
// but if we have big number of registers, this branch can be faster.
#define REP4(mac, dest, src, a, b, c, d) \
mac(dest, src, a) \
mac(dest, src, b) \
mac(dest, src, c) \
mac(dest, src, d)
#define G2(index, a, b, c, d, x, y) \
ADD_SIGMA (a, (index) + 2 * 0) \
ADD_SIGMA (b, (index) + 2 * 1) \
ADD_SIGMA (c, (index) + 2 * 2) \
ADD_SIGMA (d, (index) + 2 * 3) \
REP4 (ADD32M, 0, 1, a, b, c, d) \
REP4 (XOR32M, 3, 0, a, b, c, d) \
REP4 (RTR32M, 3, x, a, b, c, d) \
REP4 (ADD32M, 2, 3, a, b, c, d) \
REP4 (XOR32M, 1, 2, a, b, c, d) \
REP4 (RTR32M, 1, y, a, b, c, d) \
#define G(a, b, c, d) \
G2(a * 2 , a, b, c, d, 16, 12) \
G2(a * 2 + 1, a, b, c, d, 8, 7) \
#define R2 \
G(0, 1, 2, 3) \
G(4, 5, 6, 7) \
#endif
#define R(r) { const Byte *sigma = k_Blake2s_Sigma_4[r]; R2 }
// Z7_BLAKE2S_UNROLL gives 5-6 KB larger code, but faster:
// 20-40% faster for (x86/x64) VC2010+/GCC/CLANG.
// 30-60% faster for (arm64-arm32) GCC.
// 5-11% faster for (arm64) CLANG-MAC.
// so Z7_BLAKE2S_UNROLL is good optimization, if there is no vector branch.
// But if there is vectors branch (for x86*), this scalar code will be unused mostly.
// So we want smaller code (without unrolling) in that case (x86*).
#if 0 || 1 && !defined(Z7_BLAKE2S_USE_VECTORS)
#define Z7_BLAKE2S_UNROLL
#endif
#ifdef Z7_BLAKE2S_UNROLL
ROUNDS_LOOP_UNROLLED (R)
#else
ROUNDS_LOOP (R)
#endif
#undef G
#undef G2
#undef R
#undef R2
// printf("\n v after: \n");
// PrintStates2((const UInt32 *)v, 1, 16);
#define XOR_s_PAIR_v(i) s[i] ^= v[i] ^ v[i + 8];
LOOP_8(XOR_s_PAIR_v)
// printf("\n s after:\n");
// PrintStates2((const UInt32 *)s, 1, 16);
}
static
Z7_NO_INLINE
void
Z7_FASTCALL
Blake2sp_Compress2(UInt32 *s_items, const Byte *data, const Byte *end)
{
size_t pos = 0;
// PrintStates2(s_items, 8, 16);
do
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
Blake2s_Compress(s, data);
data += Z7_BLAKE2S_BLOCK_SIZE;
pos += Z7_BLAKE2S_BLOCK_SIZE;
pos &= SUPER_BLOCK_MASK;
}
while (data != end);
}
#ifdef Z7_BLAKE2S_USE_VECTORS
static Z7_BLAKE2SP_FUNC_COMPRESS g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast = Blake2sp_Compress2;
static Z7_BLAKE2SP_FUNC_COMPRESS g_Z7_BLAKE2SP_FUNC_COMPRESS_Single = Blake2sp_Compress2;
static Z7_BLAKE2SP_FUNC_INIT g_Z7_BLAKE2SP_FUNC_INIT_Init;
static Z7_BLAKE2SP_FUNC_INIT g_Z7_BLAKE2SP_FUNC_INIT_Final;
static unsigned g_z7_Blake2sp_SupportedFlags;
#define Z7_BLAKE2SP_Compress_Fast(p) (p)->u.header.func_Compress_Fast
#define Z7_BLAKE2SP_Compress_Single(p) (p)->u.header.func_Compress_Single
#else
#define Z7_BLAKE2SP_Compress_Fast(p) Blake2sp_Compress2
#define Z7_BLAKE2SP_Compress_Single(p) Blake2sp_Compress2
#endif // Z7_BLAKE2S_USE_VECTORS
#if 1 && defined(MY_CPU_LE)
#define GET_DIGEST(_s, _digest) \
{ memcpy(_digest, _s, Z7_BLAKE2S_DIGEST_SIZE); }
#else
#define GET_DIGEST(_s, _digest) \
{ unsigned _i; for (_i = 0; _i < 8; _i++) \
{ SetUi32((_digest) + 4 * _i, (_s)[_i]) } \
}
#endif
/* ---------- BLAKE2s ---------- */
/*
// we need to xor CBlake2s::h[i] with input parameter block after Blake2s_Init0()
typedef struct
{
Byte digest_length;
Byte key_length;
Byte fanout; // = 1 : in sequential mode
Byte depth; // = 1 : in sequential mode
UInt32 leaf_length;
Byte node_offset[6]; // 0 for the first, leftmost, leaf, or in sequential mode
Byte node_depth; // 0 for the leaves, or in sequential mode
Byte inner_length; // [0, 32], 0 in sequential mode
Byte salt[BLAKE2S_SALTBYTES];
Byte personal[BLAKE2S_PERSONALBYTES];
} CBlake2sParam;
*/
#define k_Blake2sp_IV_0 \
(KIV(0) ^ (Z7_BLAKE2S_DIGEST_SIZE | ((UInt32)Z7_BLAKE2SP_PARALLEL_DEGREE << 16) | ((UInt32)2 << 24)))
#define k_Blake2sp_IV_3_FROM_NODE_DEPTH(node_depth) \
(KIV(3) ^ ((UInt32)(node_depth) << 16) ^ ((UInt32)Z7_BLAKE2S_DIGEST_SIZE << 24))
Z7_FORCE_INLINE
static void Blake2sp_Init_Spec(UInt32 *s, unsigned node_offset, unsigned node_depth)
{
s[0] = k_Blake2sp_IV_0;
s[1] = KIV(1);
s[2] = KIV(2) ^ (UInt32)node_offset;
s[3] = k_Blake2sp_IV_3_FROM_NODE_DEPTH(node_depth);
s[4] = KIV(4);
s[5] = KIV(5);
s[6] = KIV(6);
s[7] = KIV(7);
STATE_T(s)[0] = 0;
STATE_T(s)[1] = 0;
STATE_F(s)[0] = 0;
STATE_F(s)[1] = 0;
}
#ifdef Z7_BLAKE2S_USE_V128_FAST
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_128BIT
BLAKE2S_ATTRIB_128BIT
#endif
void
Z7_FASTCALL
Blake2sp_InitState_V128_Fast(UInt32 *states)
{
#define STORE_128_PAIR_INIT_STATES_2(i, t0, t1) \
{ STORE_128_TO_STRUCT(states + 0 + 4 * (i), (t0)); \
STORE_128_TO_STRUCT(states + 32 + 4 * (i), (t1)); \
}
#define STORE_128_PAIR_INIT_STATES_1(i, mac) \
{ const __m128i t = mac; \
STORE_128_PAIR_INIT_STATES_2(i, t, t) \
}
#define STORE_128_PAIR_INIT_STATES_IV(i) \
STORE_128_PAIR_INIT_STATES_1(i, GET_128_IV_WAY4(i))
STORE_128_PAIR_INIT_STATES_1 (0, _mm_set1_epi32((Int32)k_Blake2sp_IV_0))
STORE_128_PAIR_INIT_STATES_IV (1)
{
const __m128i t = GET_128_IV_WAY4(2);
STORE_128_PAIR_INIT_STATES_2 (2,
XOR_128(t, _mm_set_epi32(3, 2, 1, 0)),
XOR_128(t, _mm_set_epi32(7, 6, 5, 4)))
}
STORE_128_PAIR_INIT_STATES_1 (3, _mm_set1_epi32((Int32)k_Blake2sp_IV_3_FROM_NODE_DEPTH(0)))
STORE_128_PAIR_INIT_STATES_IV (4)
STORE_128_PAIR_INIT_STATES_IV (5)
STORE_128_PAIR_INIT_STATES_IV (6)
STORE_128_PAIR_INIT_STATES_IV (7)
STORE_128_PAIR_INIT_STATES_1 (16, _mm_set_epi32(0, 0, 0, 0))
// printf("\n== exit Blake2sp_InitState_V128_Fast ctr=%d\n", states[64]);
}
#endif // Z7_BLAKE2S_USE_V128_FAST
#ifdef Z7_BLAKE2S_USE_AVX2_FAST
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_AVX2
BLAKE2S_ATTRIB_AVX2
#endif
void
Z7_FASTCALL
Blake2sp_InitState_AVX2_Fast(UInt32 *states)
{
#define STORE_256_INIT_STATES(i, t) \
STORE_256_TO_STRUCT(states + 8 * (i), t);
#define STORE_256_INIT_STATES_IV(i) \
STORE_256_INIT_STATES(i, GET_256_IV_WAY8(i))
STORE_256_INIT_STATES (0, _mm256_set1_epi32((Int32)k_Blake2sp_IV_0))
STORE_256_INIT_STATES_IV (1)
STORE_256_INIT_STATES (2, XOR_256( GET_256_IV_WAY8(2),
_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)))
STORE_256_INIT_STATES (3, _mm256_set1_epi32((Int32)k_Blake2sp_IV_3_FROM_NODE_DEPTH(0)))
STORE_256_INIT_STATES_IV (4)
STORE_256_INIT_STATES_IV (5)
STORE_256_INIT_STATES_IV (6)
STORE_256_INIT_STATES_IV (7)
STORE_256_INIT_STATES (8, _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, 0))
// printf("\n== exit Blake2sp_InitState_AVX2_Fast\n");
}
#endif // Z7_BLAKE2S_USE_AVX2_FAST
Z7_NO_INLINE
void Blake2sp_InitState(CBlake2sp *p)
{
size_t i;
// memset(p->states, 0, sizeof(p->states)); // for debug
p->u.header.cycPos = 0;
#ifdef Z7_BLAKE2SP_USE_FUNCTIONS
if (p->u.header.func_Init)
{
p->u.header.func_Init(p->states);
return;
}
#endif
for (i = 0; i < Z7_BLAKE2SP_PARALLEL_DEGREE; i++)
Blake2sp_Init_Spec(p->states + i * NSW, (unsigned)i, 0);
}
void Blake2sp_Init(CBlake2sp *p)
{
#ifdef Z7_BLAKE2SP_USE_FUNCTIONS
p->u.header.func_Compress_Fast =
#ifdef Z7_BLAKE2S_USE_VECTORS
g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast;
#else
NULL;
#endif
p->u.header.func_Compress_Single =
#ifdef Z7_BLAKE2S_USE_VECTORS
g_Z7_BLAKE2SP_FUNC_COMPRESS_Single;
#else
NULL;
#endif
p->u.header.func_Init =
#ifdef Z7_BLAKE2S_USE_VECTORS
g_Z7_BLAKE2SP_FUNC_INIT_Init;
#else
NULL;
#endif
p->u.header.func_Final =
#ifdef Z7_BLAKE2S_USE_VECTORS
g_Z7_BLAKE2SP_FUNC_INIT_Final;
#else
NULL;
#endif
#endif
Blake2sp_InitState(p);
}
void Blake2sp_Update(CBlake2sp *p, const Byte *data, size_t size)
{
size_t pos;
// printf("\nsize = 0x%6x, cycPos = %5u data = %p\n", (unsigned)size, (unsigned)p->u.header.cycPos, data);
if (size == 0)
return;
pos = p->u.header.cycPos;
// pos < SUPER_BLOCK_SIZE * 2 : is expected
// pos == SUPER_BLOCK_SIZE * 2 : is not expected, but is supported also
{
const size_t pos2 = pos & SUPER_BLOCK_MASK;
if (pos2)
{
const size_t rem = SUPER_BLOCK_SIZE - pos2;
if (rem > size)
{
p->u.header.cycPos = (unsigned)(pos + size);
// cycPos < SUPER_BLOCK_SIZE * 2
memcpy((Byte *)(void *)p->buf32 + pos, data, size);
/* to simpilify the code here we don't try to process first superblock,
if (cycPos > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) */
return;
}
// (rem <= size)
memcpy((Byte *)(void *)p->buf32 + pos, data, rem);
pos += rem;
data += rem;
size -= rem;
}
}
// pos <= SUPER_BLOCK_SIZE * 2
// pos % SUPER_BLOCK_SIZE == 0
if (pos)
{
/* pos == SUPER_BLOCK_SIZE ||
pos == SUPER_BLOCK_SIZE * 2 */
size_t end = pos;
if (size > SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE
|| (end -= SUPER_BLOCK_SIZE))
{
Z7_BLAKE2SP_Compress_Fast(p)(p->states,
(const Byte *)(const void *)p->buf32,
(const Byte *)(const void *)p->buf32 + end);
if (pos -= end)
memcpy(p->buf32, (const Byte *)(const void *)p->buf32
+ SUPER_BLOCK_SIZE, SUPER_BLOCK_SIZE);
}
}
// pos == 0 || (pos == SUPER_BLOCK_SIZE && size <= SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE)
if (size > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE)
{
// pos == 0
const Byte *end;
const size_t size2 = (size - (SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE + 1))
& ~(size_t)SUPER_BLOCK_MASK;
size -= size2;
// size < SUPER_BLOCK_SIZE * 2
end = data + size2;
Z7_BLAKE2SP_Compress_Fast(p)(p->states, data, end);
data = end;
}
if (size != 0)
{
memcpy((Byte *)(void *)p->buf32 + pos, data, size);
pos += size;
}
p->u.header.cycPos = (unsigned)pos;
// cycPos < SUPER_BLOCK_SIZE * 2
}
void Blake2sp_Final(CBlake2sp *p, Byte *digest)
{
// UInt32 * const R_states = p->states;
// printf("\nBlake2sp_Final \n");
#ifdef Z7_BLAKE2SP_USE_FUNCTIONS
if (p->u.header.func_Final)
p->u.header.func_Final(p->states);
#endif
// printf("\n=====\nBlake2sp_Final \n");
// PrintStates(p->states, 32);
// (p->u.header.cycPos == SUPER_BLOCK_SIZE) can be processed in any branch:
if (p->u.header.cycPos <= SUPER_BLOCK_SIZE)
{
unsigned pos;
memset((Byte *)(void *)p->buf32 + p->u.header.cycPos,
0, SUPER_BLOCK_SIZE - p->u.header.cycPos);
STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG;
for (pos = 0; pos < SUPER_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE)
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos);
Blake2s_Set_LastBlock(s)
if (pos + Z7_BLAKE2S_BLOCK_SIZE > p->u.header.cycPos)
{
UInt32 delta = Z7_BLAKE2S_BLOCK_SIZE;
if (pos < p->u.header.cycPos)
delta -= p->u.header.cycPos & (Z7_BLAKE2S_BLOCK_SIZE - 1);
// 0 < delta <= Z7_BLAKE2S_BLOCK_SIZE
{
const UInt32 v = STATE_T(s)[0];
STATE_T(s)[1] -= v < delta; // (v < delta) is same condition here as (v == 0)
STATE_T(s)[0] = v - delta;
}
}
}
// PrintStates(p->states, 16);
Z7_BLAKE2SP_Compress_Single(p)(p->states,
(Byte *)(void *)p->buf32,
(Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE);
// PrintStates(p->states, 16);
}
else
{
// (p->u.header.cycPos > SUPER_BLOCK_SIZE)
unsigned pos;
for (pos = 0; pos < SUPER_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE)
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos);
if (pos + SUPER_BLOCK_SIZE >= p->u.header.cycPos)
Blake2s_Set_LastBlock(s)
}
if (p->u.header.cycPos <= SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE)
STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG;
Z7_BLAKE2SP_Compress_Single(p)(p->states,
(Byte *)(void *)p->buf32,
(Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE);
// if (p->u.header.cycPos > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE;
STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG;
// if (p->u.header.cycPos != SUPER_BLOCK_SIZE)
{
pos = SUPER_BLOCK_SIZE;
for (;;)
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos & SUPER_BLOCK_MASK);
Blake2s_Set_LastBlock(s)
pos += Z7_BLAKE2S_BLOCK_SIZE;
if (pos >= p->u.header.cycPos)
{
if (pos != p->u.header.cycPos)
{
const UInt32 delta = pos - p->u.header.cycPos;
const UInt32 v = STATE_T(s)[0];
STATE_T(s)[1] -= v < delta;
STATE_T(s)[0] = v - delta;
memset((Byte *)(void *)p->buf32 + p->u.header.cycPos, 0, delta);
}
break;
}
}
Z7_BLAKE2SP_Compress_Single(p)(p->states,
(Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE,
(Byte *)(void *)p->buf32 + pos);
}
}
{
size_t pos;
for (pos = 0; pos < SUPER_BLOCK_SIZE / 2; pos += Z7_BLAKE2S_BLOCK_SIZE / 2)
{
const UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, (pos * 2));
Byte *dest = (Byte *)(void *)p->buf32 + pos;
GET_DIGEST(s, dest)
}
}
Blake2sp_Init_Spec(p->states, 0, 1);
{
size_t pos;
for (pos = 0; pos < (Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2S_DIGEST_SIZE)
- Z7_BLAKE2S_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE)
{
Z7_BLAKE2SP_Compress_Single(p)(p->states,
(const Byte *)(const void *)p->buf32 + pos,
(const Byte *)(const void *)p->buf32 + pos + Z7_BLAKE2S_BLOCK_SIZE);
}
}
// Blake2s_Final(p->states, 0, digest, p, (Byte *)(void *)p->buf32 + i);
Blake2s_Set_LastBlock(p->states)
STATE_F(p->states)[1] = BLAKE2S_FINAL_FLAG;
{
Z7_BLAKE2SP_Compress_Single(p)(p->states,
(const Byte *)(const void *)p->buf32 + Z7_BLAKE2SP_PARALLEL_DEGREE / 2 * Z7_BLAKE2S_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE,
(const Byte *)(const void *)p->buf32 + Z7_BLAKE2SP_PARALLEL_DEGREE / 2 * Z7_BLAKE2S_BLOCK_SIZE);
}
GET_DIGEST(p->states, digest)
// printf("\n Blake2sp_Final 555 numDataInBufs = %5u\n", (unsigned)p->u.header.numDataInBufs);
}
BoolInt Blake2sp_SetFunction(CBlake2sp *p, unsigned algo)
{
// printf("\n========== setfunction = %d ======== \n", algo);
#ifdef Z7_BLAKE2SP_USE_FUNCTIONS
Z7_BLAKE2SP_FUNC_COMPRESS func = NULL;
Z7_BLAKE2SP_FUNC_COMPRESS func_Single = NULL;
Z7_BLAKE2SP_FUNC_INIT func_Final = NULL;
Z7_BLAKE2SP_FUNC_INIT func_Init = NULL;
#else
UNUSED_VAR(p)
#endif
#ifdef Z7_BLAKE2S_USE_VECTORS
func = func_Single = Blake2sp_Compress2;
if (algo != Z7_BLAKE2SP_ALGO_SCALAR)
{
// printf("\n========== setfunction NON-SCALER ======== \n");
if (algo == Z7_BLAKE2SP_ALGO_DEFAULT)
{
func = g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast;
func_Single = g_Z7_BLAKE2SP_FUNC_COMPRESS_Single;
func_Init = g_Z7_BLAKE2SP_FUNC_INIT_Init;
func_Final = g_Z7_BLAKE2SP_FUNC_INIT_Final;
}
else
{
if ((g_z7_Blake2sp_SupportedFlags & (1u << algo)) == 0)
return False;
#ifdef Z7_BLAKE2S_USE_AVX2
func_Single =
#if defined(Z7_BLAKE2S_USE_AVX2_WAY2)
Blake2sp_Compress2_AVX2_Way2;
#else
Z7_BLAKE2S_Compress2_V128;
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_FAST
if (algo == Z7_BLAKE2SP_ALGO_V256_FAST)
{
func = Blake2sp_Compress2_AVX2_Fast;
func_Final = Blake2sp_Final_AVX2_Fast;
func_Init = Blake2sp_InitState_AVX2_Fast;
}
else
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY2
if (algo == Z7_BLAKE2SP_ALGO_V256_WAY2)
func = Blake2sp_Compress2_AVX2_Way2;
else
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY4
if (algo == Z7_BLAKE2SP_ALGO_V256_WAY4)
{
func_Single = func = Blake2sp_Compress2_AVX2_Way4;
}
else
#endif
#endif // avx2
{
if (algo == Z7_BLAKE2SP_ALGO_V128_FAST)
{
func = Blake2sp_Compress2_V128_Fast;
func_Final = Blake2sp_Final_V128_Fast;
func_Init = Blake2sp_InitState_V128_Fast;
func_Single = Z7_BLAKE2S_Compress2_V128;
}
else
#ifdef Z7_BLAKE2S_USE_V128_WAY2
if (algo == Z7_BLAKE2SP_ALGO_V128_WAY2)
func = func_Single = Blake2sp_Compress2_V128_Way2;
else
#endif
{
if (algo != Z7_BLAKE2SP_ALGO_V128_WAY1)
return False;
func = func_Single = Blake2sp_Compress2_V128_Way1;
}
}
}
}
#else // !VECTORS
if (algo > 1) // Z7_BLAKE2SP_ALGO_SCALAR
return False;
#endif // !VECTORS
#ifdef Z7_BLAKE2SP_USE_FUNCTIONS
p->u.header.func_Compress_Fast = func;
p->u.header.func_Compress_Single = func_Single;
p->u.header.func_Final = func_Final;
p->u.header.func_Init = func_Init;
#endif
// printf("\n p->u.header.func_Compress = %p", p->u.header.func_Compress);
return True;
}
void z7_Black2sp_Prepare(void)
{
#ifdef Z7_BLAKE2S_USE_VECTORS
unsigned flags = 0; // (1u << Z7_BLAKE2SP_ALGO_V128_SCALAR);
Z7_BLAKE2SP_FUNC_COMPRESS func_Fast = Blake2sp_Compress2;
Z7_BLAKE2SP_FUNC_COMPRESS func_Single = Blake2sp_Compress2;
Z7_BLAKE2SP_FUNC_INIT func_Init = NULL;
Z7_BLAKE2SP_FUNC_INIT func_Final = NULL;
#if defined(MY_CPU_X86_OR_AMD64)
#if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS)
// optional check
#if 0 || !(defined(__AVX512F__) && defined(__AVX512VL__))
if (CPU_IsSupported_AVX512F_AVX512VL())
#endif
#elif defined(Z7_BLAKE2S_USE_SSE41)
if (CPU_IsSupported_SSE41())
#elif defined(Z7_BLAKE2S_USE_SSSE3)
if (CPU_IsSupported_SSSE3())
#elif !defined(MY_CPU_AMD64)
if (CPU_IsSupported_SSE2())
#endif
#endif
{
#if defined(Z7_BLAKE2S_USE_SSE41)
// printf("\n========== Blake2s SSE41 128-bit\n");
#elif defined(Z7_BLAKE2S_USE_SSSE3)
// printf("\n========== Blake2s SSSE3 128-bit\n");
#else
// printf("\n========== Blake2s SSE2 128-bit\n");
#endif
// func_Fast = f_vector = Blake2sp_Compress2_V128_Way2;
// printf("\n========== Blake2sp_Compress2_V128_Way2\n");
func_Fast =
func_Single = Z7_BLAKE2S_Compress2_V128;
flags |= (1u << Z7_BLAKE2SP_ALGO_V128_WAY1);
#ifdef Z7_BLAKE2S_USE_V128_WAY2
flags |= (1u << Z7_BLAKE2SP_ALGO_V128_WAY2);
#endif
#ifdef Z7_BLAKE2S_USE_V128_FAST
flags |= (1u << Z7_BLAKE2SP_ALGO_V128_FAST);
func_Fast = Blake2sp_Compress2_V128_Fast;
func_Init = Blake2sp_InitState_V128_Fast;
func_Final = Blake2sp_Final_V128_Fast;
#endif
#ifdef Z7_BLAKE2S_USE_AVX2
#if defined(MY_CPU_X86_OR_AMD64)
#if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS)
#if 0
if (CPU_IsSupported_AVX512F_AVX512VL())
#endif
#else
if (CPU_IsSupported_AVX2())
#endif
#endif
{
// #pragma message ("=== Blake2s AVX2")
// printf("\n========== Blake2s AVX2\n");
#ifdef Z7_BLAKE2S_USE_AVX2_WAY2
func_Single = Blake2sp_Compress2_AVX2_Way2;
flags |= (1u << Z7_BLAKE2SP_ALGO_V256_WAY2);
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY4
flags |= (1u << Z7_BLAKE2SP_ALGO_V256_WAY4);
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_FAST
flags |= (1u << Z7_BLAKE2SP_ALGO_V256_FAST);
func_Fast = Blake2sp_Compress2_AVX2_Fast;
func_Init = Blake2sp_InitState_AVX2_Fast;
func_Final = Blake2sp_Final_AVX2_Fast;
#elif defined(Z7_BLAKE2S_USE_AVX2_WAY4)
func_Fast = Blake2sp_Compress2_AVX2_Way4;
#elif defined(Z7_BLAKE2S_USE_AVX2_WAY2)
func_Fast = Blake2sp_Compress2_AVX2_Way2;
#endif
} // avx2
#endif // avx2
} // sse*
g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast = func_Fast;
g_Z7_BLAKE2SP_FUNC_COMPRESS_Single = func_Single;
g_Z7_BLAKE2SP_FUNC_INIT_Init = func_Init;
g_Z7_BLAKE2SP_FUNC_INIT_Final = func_Final;
g_z7_Blake2sp_SupportedFlags = flags;
// printf("\nflags=%x\n", flags);
#endif // vectors
}
/*
#ifdef Z7_BLAKE2S_USE_VECTORS
void align_test2(CBlake2sp *sp);
void align_test2(CBlake2sp *sp)
{
__m128i a = LOAD_128(sp->states);
D_XOR_128(a, LOAD_128(sp->states + 4));
STORE_128(sp->states, a);
}
void align_test2(void);
void align_test2(void)
{
CBlake2sp sp;
Blake2sp_Init(&sp);
Blake2sp_Update(&sp, NULL, 0);
}
#endif
*/
Reference in a new issue View git blame Copy permalink