__m256i branchfree_search8_avx(int* source, size_t n, __m256i target) {
__m256i offsets = _mm256_setzero_si256();
if(n == 0) return offsets;
__m256i ha = _mm256_set1_epi32(n>>1);
while(n>1) {
n -= n>>1;
__m256i offsetsplushalf = _mm256_add_epi32(offsets,ha);
ha = _mm256_sub_epi32(ha,_mm256_srli_epi32(ha,1));
__m256i keys = _mm256_i32gather_epi32(source,offsetsplushalf,4);
__m256i lt = _mm256_cmpgt_epi32(target,keys);
offsets = _mm256_blendv_epi8(offsets,offsetsplushalf,lt);
}
__m256i lastkeys = _mm256_i32gather_epi32(source,offsets,4);
__m256i lastlt = _mm256_cmpgt_epi32(target,lastkeys);
__m256i oneswhereneeded = _mm256_srli_epi32(lastlt,31);
__m256i answer = _mm256_add_epi32(offsets,oneswhereneeded);
return answer;
}
// credit: Harold Aptroot
uint32_t maskedvectorsum(uint32_t * z, uint32_t N, uint32_t * accesses,
uint32_t nmbr) {
__m256i Nvec = _mm256_set1_epi32(N - 1);
__m256i sum = _mm256_setzero_si256();
for(uint32_t j = 0; j < nmbr ; j += 8) {
__m256i indexes = _mm256_loadu_si256((__m256i*)(accesses + j));
indexes = _mm256_and_si256(indexes, Nvec);
__m256i fi = _mm256_i32gather_epi32((int*)z, indexes, 4);
sum = _mm256_add_epi32(sum, fi);
}
__m128i sum128 = _mm_add_epi32(_mm256_extracti128_si256(sum, 0), _mm256_extracti128_si256(sum, 1));
sum128 = _mm_hadd_epi32(sum128, sum128);
return _mm_extract_epi32(sum128, 0) + _mm_extract_epi32(sum128, 1);
}
static FORCE_INLINE __m256i lookup_double_AVX2(const int16_t *VXFull, const int16_t *VYFull, const PixelType *pref, int w, const __m256i &dwords_ref_pitch, const __m256i &dwords_hoffsets) {
__m256i vx = _mm256_cvtepi16_epi32(_mm_loadu_si128((const __m128i *)&VXFull[w]));
vx = _mm256_srai_epi32(vx, 1);
__m256i vy = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)&VYFull[w]));
vy = _mm256_srai_epi16(vy, 1);
__m256i addr = _mm256_madd_epi16(vy, dwords_ref_pitch);
addr = _mm256_add_epi32(addr, vx);
addr = _mm256_add_epi32(addr, dwords_hoffsets);
// It's okay to read two or three bytes more than needed. pref is always padded, unless the user chooses a horizontal padding of 0, which would be stupid.
__m256i gathered = _mm256_i32gather_epi32((const int *)pref, addr, sizeof(PixelType));
gathered = _mm256_and_si256(gathered, _mm256_set1_epi32((1 << (sizeof(PixelType) * 8)) - 1));
return gathered;
}
__m256i test_mm256_i32gather_epi32(int const *b, __m256i c) {
// CHECK-LABEL: test_mm256_i32gather_epi32
// CHECK: call <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> %{{.*}}, i8* %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, i8 2)
return _mm256_i32gather_epi32(b, c, 2);
}
static FORCE_INLINE void FlowInter_8px_AVX2(
int w, PixelType *pdst,
const PixelType *prefB, const PixelType *prefF,
const int16_t *VXFullB, const int16_t *VXFullF,
const int16_t *VYFullB, const int16_t *VYFullF,
const uint8_t *MaskB, const uint8_t *MaskF,
int nPelLog,
const __m256i &dwords_time256, const __m256i &dwords_256_time256,
const __m256i &dwords_ref_pitch, const __m256i &dwords_hoffsets) {
__m256i dwords_w = _mm256_add_epi32(_mm256_set1_epi32(w << nPelLog), dwords_hoffsets);
__m256i dstF = lookup_AVX2(VXFullF, VYFullF, prefF, w, dwords_time256, dwords_ref_pitch, dwords_w);
__m256i dstB = lookup_AVX2(VXFullB, VYFullB, prefB, w, dwords_256_time256, dwords_ref_pitch, dwords_w);
__m256i dstF0 = _mm256_i32gather_epi32((const int *)prefF, dwords_w, sizeof(PixelType));
__m256i dstB0 = _mm256_i32gather_epi32((const int *)prefB, dwords_w, sizeof(PixelType));
dstF0 = _mm256_and_si256(dstF0, _mm256_set1_epi32((1 << (sizeof(PixelType) * 8)) - 1));
dstB0 = _mm256_and_si256(dstB0, _mm256_set1_epi32((1 << (sizeof(PixelType) * 8)) - 1));
__m256i maskf = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskF[w]));
__m256i maskb = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskB[w]));
const __m256i dwords_255 = _mm256_set1_epi32(255);
__m256i maskf_inv = _mm256_sub_epi32(dwords_255, maskf);
__m256i maskb_inv = _mm256_sub_epi32(dwords_255, maskb);
__m256i dstF_maskf_inv, dstB_maskb_inv, dstF0_maskb, dstB0_maskf;
if (sizeof(PixelType) == 1) {
dstF_maskf_inv = _mm256_mullo_epi16(dstF, maskf_inv);
dstB_maskb_inv = _mm256_mullo_epi16(dstB, maskb_inv);
dstF0_maskb = _mm256_mullo_epi16(dstF0, maskb);
dstB0_maskf = _mm256_mullo_epi16(dstB0, maskf);
} else {
dstF_maskf_inv = _mm256_mullo_epi32(dstF, maskf_inv);
dstB_maskb_inv = _mm256_mullo_epi32(dstB, maskb_inv);
dstF0_maskb = _mm256_mullo_epi32(dstF0, maskb);
dstB0_maskf = _mm256_mullo_epi32(dstB0, maskf);
}
__m256i f = _mm256_add_epi32(dstF0_maskb, dstB_maskb_inv);
__m256i b = _mm256_add_epi32(dstB0_maskf, dstF_maskf_inv);
if (sizeof(PixelType) == 1) {
f = _mm256_mullo_epi32(f, maskf);
b = _mm256_mullo_epi32(b, maskb);
f = _mm256_add_epi32(f, dwords_255);
b = _mm256_add_epi32(b, dwords_255);
f = _mm256_srai_epi32(f, 8);
b = _mm256_srai_epi32(b, 8);
} else {
const __m256i qwords_255 = _mm256_set1_epi64x(255);
__m256i tempf = _mm256_mul_epu32(f, maskf);
__m256i tempb = _mm256_mul_epu32(b, maskb);
tempf = _mm256_add_epi64(tempf, qwords_255);
tempb = _mm256_add_epi64(tempb, qwords_255);
tempf = _mm256_srli_epi64(tempf, 8);
tempb = _mm256_srli_epi64(tempb, 8);
f = _mm256_srli_epi64(f, 32);
b = _mm256_srli_epi64(b, 32);
f = _mm256_mul_epu32(f, _mm256_srli_epi64(maskf, 32));
b = _mm256_mul_epu32(b, _mm256_srli_epi64(maskb, 32));
f = _mm256_add_epi64(f, qwords_255);
b = _mm256_add_epi64(b, qwords_255);
f = _mm256_srli_epi64(f, 8);
b = _mm256_srli_epi64(b, 8);
f = _mm256_or_si256(tempf, _mm256_slli_epi64(f, 32));
b = _mm256_or_si256(tempb, _mm256_slli_epi64(b, 32));
}
f = _mm256_add_epi32(f, dstF_maskf_inv);
b = _mm256_add_epi32(b, dstB_maskb_inv);
f = _mm256_add_epi32(f, dwords_255);
b = _mm256_add_epi32(b, dwords_255);
f = _mm256_srai_epi32(f, 8);
b = _mm256_srai_epi32(b, 8);
if (sizeof(PixelType) == 1) {
f = _mm256_madd_epi16(f, dwords_256_time256);
b = _mm256_madd_epi16(b, dwords_time256);
} else {
f = _mm256_mullo_epi32(f, dwords_256_time256);
b = _mm256_mullo_epi32(b, dwords_time256);
}
__m256i dst = _mm256_add_epi32(f, b);
dst = _mm256_srai_epi32(dst, 8);
dst = _mm256_packus_epi32(dst, dst);
dst = _mm256_permute4x64_epi64(dst, 0xe8); // 0b11101000 - copy third qword to second qword
__m128i dst128 = _mm256_castsi256_si128(dst);
if (sizeof(PixelType) == 1) {
dst128 = _mm_packus_epi16(dst128, dst128);
_mm_storel_epi64((__m128i *)&pdst[w], dst128);
} else {
_mm_storeu_si128((__m128i *)&pdst[w], dst128);
}
}
__m256i test_mm256_i32gather_epi32(int const *b, __m256i c) {
// CHECK: @llvm.x86.avx2.gather.d.d.256
return _mm256_i32gather_epi32(b, c, 2);
}
