__m128i test_mm_sll_epi64(__m128i A, __m128i B) {
// DAG-LABEL: test_mm_sll_epi64
// DAG: call <2 x i64> @llvm.x86.sse2.psll.q
//
// ASM-LABEL: test_mm_sll_epi64
// ASM: psllq
return _mm_sll_epi64(A, B);
}
/// get the number of non-zero
size_t vec_i8_cnt_nonzero(const int8_t *p, size_t n)
{
size_t ans = 0;
#ifdef COREARRAY_SIMD_SSE2
const __m128i ZERO = { 0LL, 0LL };
const __m128i ONES = { 0x0101010101010101LL, 0x0101010101010101LL };
const __m128i ONE = { 1LL, 1LL };
// header 1, 16-byte aligned
size_t h = (16 - ((size_t)p & 0x0F)) & 0x0F;
for (; (n > 0) && (h > 0); n--, h--)
ans += (*p++) ? 1 : 0;
# ifdef COREARRAY_SIMD_AVX2
// header 2, 32-byte aligned
if ((n >= 16) && ((size_t)p & 0x10))
{
__m128i c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
__m128i bit = _mm_and_si128(c, ONES);
p += 16; n -= 16;
uint64_t array[2] __attribute__((aligned(16)));
*((__m128i*)array) = bit;
ans += 16 - POPCNT_U64(array[0]) - POPCNT_U64(array[1]);
}
const __m256i ZERO2 = { 0LL, 0LL, 0LL, 0LL };
const __m256i ONES2 = { 0x0101010101010101LL, 0x0101010101010101LL,
0x0101010101010101LL, 0x0101010101010101LL };
// body, AVX2
for (; n >= 256; n -= 256)
{
__m256i c = _mm256_cmpeq_epi8(_mm256_load_si256((__m256i const*)p), ZERO2);
__m256i bit = _mm256_and_si256(c, ONES2);
p += 32;
c = _mm256_cmpeq_epi8(_mm256_load_si256((__m256i const*)p), ZERO2);
bit = _mm256_or_si256(_mm256_sll_epi64(bit, ONE), _mm256_and_si256(c, ONES2));
p += 32;
c = _mm256_cmpeq_epi8(_mm256_load_si256((__m256i const*)p), ZERO2);
bit = _mm256_or_si256(_mm256_sll_epi64(bit, ONE), _mm256_and_si256(c, ONES2));
p += 32;
c = _mm256_cmpeq_epi8(_mm256_load_si256((__m256i const*)p), ZERO2);
bit = _mm256_or_si256(_mm256_sll_epi64(bit, ONE), _mm256_and_si256(c, ONES2));
p += 32;
c = _mm256_cmpeq_epi8(_mm256_load_si256((__m256i const*)p), ZERO2);
bit = _mm256_or_si256(_mm256_sll_epi64(bit, ONE), _mm256_and_si256(c, ONES2));
p += 32;
c = _mm256_cmpeq_epi8(_mm256_load_si256((__m256i const*)p), ZERO2);
bit = _mm256_or_si256(_mm256_sll_epi64(bit, ONE), _mm256_and_si256(c, ONES2));
p += 32;
c = _mm256_cmpeq_epi8(_mm256_load_si256((__m256i const*)p), ZERO2);
bit = _mm256_or_si256(_mm256_sll_epi64(bit, ONE), _mm256_and_si256(c, ONES2));
p += 32;
c = _mm256_cmpeq_epi8(_mm256_load_si256((__m256i const*)p), ZERO2);
bit = _mm256_or_si256(_mm256_sll_epi64(bit, ONE), _mm256_and_si256(c, ONES2));
p += 32;
uint64_t array[4] __attribute__((aligned(32)));
*((__m256i*)array) = bit;
ans += 256 - POPCNT_U64(array[0]) - POPCNT_U64(array[1]) -
POPCNT_U64(array[2]) - POPCNT_U64(array[3]);
}
# endif
// body, SSE2
for (; n >= 128; n -= 128)
{
__m128i c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
__m128i bit = _mm_and_si128(c, ONES);
p += 16;
c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
bit = _mm_or_si128(_mm_sll_epi64(bit, ONE), _mm_and_si128(c, ONES));
p += 16;
c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
bit = _mm_or_si128(_mm_sll_epi64(bit, ONE), _mm_and_si128(c, ONES));
p += 16;
c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
bit = _mm_or_si128(_mm_sll_epi64(bit, ONE), _mm_and_si128(c, ONES));
p += 16;
c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
bit = _mm_or_si128(_mm_sll_epi64(bit, ONE), _mm_and_si128(c, ONES));
p += 16;
c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
bit = _mm_or_si128(_mm_sll_epi64(bit, ONE), _mm_and_si128(c, ONES));
p += 16;
c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
bit = _mm_or_si128(_mm_sll_epi64(bit, ONE), _mm_and_si128(c, ONES));
p += 16;
c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
bit = _mm_or_si128(_mm_sll_epi64(bit, ONE), _mm_and_si128(c, ONES));
p += 16;
uint64_t array[2] __attribute__((aligned(16)));
*((__m128i*)array) = bit;
ans += 128 - POPCNT_U64(array[0]) - POPCNT_U64(array[1]);
}
for (; n >= 16; n -= 16)
{
__m128i c = _mm_cmpeq_epi8(_mm_load_si128((__m128i const*)p), ZERO);
__m128i bit = _mm_and_si128(c, ONES);
p += 16;
uint64_t array[2] __attribute__((aligned(16)));
*((__m128i*)array) = bit;
ans += 16 - POPCNT_U64(array[0]) - POPCNT_U64(array[1]);
}
#else
// header, 8-byte aligned
size_t h = (8 - ((size_t)p & 0x07)) & 0x07;
for (; (n > 0) && (h > 0); n--, h--)
ans += (*p++) ? 1 : 0;
// body, unroll
for (; n >= 8; n -= 8)
{
ans += (p[0] ? 1 : 0) + (p[1] ? 1 : 0) +
(p[2] ? 1 : 0) + (p[3] ? 1 : 0) +
(p[4] ? 1 : 0) + (p[5] ? 1 : 0) +
(p[6] ? 1 : 0) + (p[7] ? 1 : 0);
p += 8;
}
#endif
// tail
for (; n > 0; n--) ans += (*p++) ? 1 : 0;
return ans;
}
sse2_tests (void)
{
/* psraw */
c128.v = _mm_srai_epi16 (m128_16, SHIFT);
dump128_16 (buf, "_mm_srai_epi16", c128);
c128.v = _mm_sra_epi16 (m128_16, s128);
dump128_16 (buf, "_mm_sra_epi16", c128);
/* psrad */
c128.v = _mm_srai_epi32 (m128_32, SHIFT);
dump128_32 (buf, "_mm_srai_epi32", c128);
c128.v = _mm_sra_epi32 (m128_32, s128);
dump128_32 (buf, "_mm_sra_epi32", c128);
/* psrlw */
c128.v = _mm_srli_epi16 (m128_16, SHIFT);
dump128_16 (buf, "_mm_srli_epi16", c128);
c128.v = _mm_srl_epi16 (m128_16, s128);
dump128_16 (buf, "_mm_srl_epi16", c128);
/* psrld */
c128.v = _mm_srli_epi32 (m128_32, SHIFT);
dump128_32 (buf, "_mm_srli_epi32", c128);
c128.v = _mm_srl_epi32 (m128_32, s128);
dump128_32 (buf, "_mm_srl_epi32", c128);
/* psrlq */
c128.v = _mm_srli_epi64 (m128_64, SHIFT);
dump128_64 (buf, "_mm_srli_epi64", c128);
c128.v = _mm_srl_epi64 (m128_64, s128);
dump128_64 (buf, "_mm_srl_epi64", c128);
/* psrldq */
c128.v = _mm_srli_si128 (m128_128, SHIFT);
dump128_128 (buf, "_mm_srli_si128 (byte shift) ", c128);
/* psllw */
c128.v = _mm_slli_epi16 (m128_16, SHIFT);
dump128_16 (buf, "_mm_slli_epi16", c128);
c128.v = _mm_sll_epi16 (m128_16, s128);
dump128_16 (buf, "_mm_sll_epi16", c128);
/* pslld */
c128.v = _mm_slli_epi32 (m128_32, SHIFT);
dump128_32 (buf, "_mm_slli_epi32", c128);
c128.v = _mm_sll_epi32 (m128_32, s128);
dump128_32 (buf, "_mm_sll_epi32", c128);
/* psllq */
c128.v = _mm_slli_epi64 (m128_64, SHIFT);
dump128_64 (buf, "_mm_slli_epi64", c128);
c128.v = _mm_sll_epi64 (m128_64, s128);
dump128_64 (buf, "_mm_sll_epi64", c128);
/* pslldq */
c128.v = _mm_slli_si128 (m128_128, SHIFT);
dump128_128 (buf, "_mm_sll_si128 (byte shift)", c128);
/* Shuffle constant 0x1b == 0b_00_01_10_11, e.g. swap words: ABCD => DCBA. */
/* pshufd */
c128.v = _mm_shuffle_epi32 (m128_128, 0x1b);
dump128_32 (buf, "_mm_shuffle_epi32", c128);
/* pshuflw */
c128.v = _mm_shufflelo_epi16 (m128_128, 0x1b);
dump128_16 (buf, "_mm_shuffelo_epi16", c128);
/* pshufhw */
c128.v = _mm_shufflehi_epi16 (m128_128, 0x1b);
dump128_16 (buf, "_mm_shuffehi_epi16", c128);
}
