int32_t avx2_sumsignedbytes_variant2(int8_t* array, size_t size) {
__m256i accumulator = _mm256_setzero_si256();
for (size_t i=0; i < size; i += 32) {
const __m256i v = _mm256_loadu_si256((__m256i*)(array + i));
const __m256i v0 = _mm256_srai_epi32(v, 3*8);
const __m256i v1 = _mm256_srai_epi32(_mm256_slli_epi32(v, 1*8), 3*8);
const __m256i v2 = _mm256_srai_epi32(_mm256_slli_epi32(v, 2*8), 3*8);
const __m256i v3 = _mm256_srai_epi32(_mm256_slli_epi32(v, 3*8), 3*8);
accumulator = _mm256_add_epi32(accumulator, v0);
accumulator = _mm256_add_epi32(accumulator, v1);
accumulator = _mm256_add_epi32(accumulator, v2);
accumulator = _mm256_add_epi32(accumulator, v3);
}
return int32_t(_mm256_extract_epi32(accumulator, 0)) +
int32_t(_mm256_extract_epi32(accumulator, 1)) +
int32_t(_mm256_extract_epi32(accumulator, 2)) +
int32_t(_mm256_extract_epi32(accumulator, 3)) +
int32_t(_mm256_extract_epi32(accumulator, 4)) +
int32_t(_mm256_extract_epi32(accumulator, 5)) +
int32_t(_mm256_extract_epi32(accumulator, 6)) +
int32_t(_mm256_extract_epi32(accumulator, 7));
}
__m256i Scaler::process_vect_int_avx2 (const __m256i &add_cst, int kernel_size, const __m256i coef_base_ptr [], typename SRC::PtrConst::Type pix_ptr, const __m256i &zero, int src_stride, const __m256i &sign_bit, int len)
{
typedef typename SRC::template S16 <false, (SB == 16)> SrcS16R;
__m256i sum0 = add_cst;
__m256i sum1 = add_cst;
for (int k = 0; k < kernel_size; ++k)
{
const __m256i coef = _mm256_load_si256 (coef_base_ptr + k);
const __m256i src = ReadWrapperInt <SRC, SrcS16R, PF>::read (
pix_ptr, zero, sign_bit, len
);
fstb::ToolsAvx2::mac_s16_s16_s32 (sum0, sum1, src, coef);
SRC::PtrConst::jump (pix_ptr, src_stride);
}
sum0 = _mm256_srai_epi32 (sum0, SHIFT_INT + SB - DB);
sum1 = _mm256_srai_epi32 (sum1, SHIFT_INT + SB - DB);
const __m256i val = _mm256_packs_epi32 (sum0, sum1);
return (val);
}
static INLINE unsigned int highbd_masked_sad16xh_avx2(
const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
const uint8_t *b8, int b_stride, const uint8_t *m_ptr, int m_stride,
int width, int height) {
const uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src8);
const uint16_t *a_ptr = CONVERT_TO_SHORTPTR(a8);
const uint16_t *b_ptr = CONVERT_TO_SHORTPTR(b8);
int x, y;
__m256i res = _mm256_setzero_si256();
const __m256i mask_max = _mm256_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
const __m256i round_const =
_mm256_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
const __m256i one = _mm256_set1_epi16(1);
for (y = 0; y < height; y++) {
for (x = 0; x < width; x += 16) {
const __m256i src = _mm256_lddqu_si256((const __m256i *)&src_ptr[x]);
const __m256i a = _mm256_lddqu_si256((const __m256i *)&a_ptr[x]);
const __m256i b = _mm256_lddqu_si256((const __m256i *)&b_ptr[x]);
// Zero-extend mask to 16 bits
const __m256i m =
_mm256_cvtepu8_epi16(_mm_lddqu_si128((const __m128i *)&m_ptr[x]));
const __m256i m_inv = _mm256_sub_epi16(mask_max, m);
const __m256i data_l = _mm256_unpacklo_epi16(a, b);
const __m256i mask_l = _mm256_unpacklo_epi16(m, m_inv);
__m256i pred_l = _mm256_madd_epi16(data_l, mask_l);
pred_l = _mm256_srai_epi32(_mm256_add_epi32(pred_l, round_const),
AOM_BLEND_A64_ROUND_BITS);
const __m256i data_r = _mm256_unpackhi_epi16(a, b);
const __m256i mask_r = _mm256_unpackhi_epi16(m, m_inv);
__m256i pred_r = _mm256_madd_epi16(data_r, mask_r);
pred_r = _mm256_srai_epi32(_mm256_add_epi32(pred_r, round_const),
AOM_BLEND_A64_ROUND_BITS);
// Note: the maximum value in pred_l/r is (2^bd)-1 < 2^15,
// so it is safe to do signed saturation here.
const __m256i pred = _mm256_packs_epi32(pred_l, pred_r);
// There is no 16-bit SAD instruction, so we have to synthesize
// an 8-element SAD. We do this by storing 4 32-bit partial SADs,
// and accumulating them at the end
const __m256i diff = _mm256_abs_epi16(_mm256_sub_epi16(pred, src));
res = _mm256_add_epi32(res, _mm256_madd_epi16(diff, one));
}
src_ptr += src_stride;
a_ptr += a_stride;
b_ptr += b_stride;
m_ptr += m_stride;
}
// At this point, we have four 32-bit partial SADs stored in 'res'.
res = _mm256_hadd_epi32(res, res);
res = _mm256_hadd_epi32(res, res);
int sad = _mm256_extract_epi32(res, 0) + _mm256_extract_epi32(res, 4);
return (sad + 31) >> 6;
}
inline void avx2_xy_to_uv_f(__m256 x, __m256 y, __m256i& u, __m256i& v)
{
// Convert X,Y first into U,V space then round to nearest
// integer. That gets us close to correct answer, mapping XY to a
// lozenge-shaped space rather than hexagonal. We then correct the
// four regions that lie outside the hexagonal cell assigning them
// to their correct neighboring cell.
// Writer's note: see ~/Google Drive/Work/calin
// double dv = y*c_vy_inv;
// double du = x-dv*c_vx;
// u = std::lround(du);
// v = std::lround(dv);
// du -= u;
// dv -= v;
y = _mm256_mul_ps(y, calin::math::simd::c_m256(_c_m256_vy_inv));
x = _mm256_fnmadd_ps(y, calin::math::simd::c_m256(_c_m256_vx), x);
u = _mm256_cvtps_epi32(x);
v = _mm256_cvtps_epi32(y);
x = _mm256_sub_ps(x, _mm256_cvtepi32_ps(u));
y = _mm256_sub_ps(y, _mm256_cvtepi32_ps(v));
// double c3 = dv-du;
const __m256i c3 = _mm256_castps_si256(_mm256_sub_ps(y, x));
__m256i uvshift;
__m256i mask;
// double c1 = du+0.5*dv;
// double c2 = dv+0.5*du;
// if(c3<0) {
// if(c1>=1) u++;
// else if(c2<-1) v--;
// } else {
// if(c2>=1) v++;
// else if(c1<-1) u--;
// }
uvshift = _mm256_cvtps_epi32(_mm256_fmadd_ps(y, calin::math::simd::c_m256(_c_m256_one_half), x));
mask = _mm256_srai_epi32(_mm256_xor_si256(uvshift, c3), 31);
u = _mm256_blendv_epi8(u, _mm256_add_epi32(u, uvshift), mask);
uvshift = _mm256_cvtps_epi32(_mm256_fmadd_ps(x, calin::math::simd::c_m256(_c_m256_one_half), y));
mask = _mm256_srai_epi32(_mm256_xor_si256(uvshift, c3), 31);
v = _mm256_blendv_epi8(_mm256_add_epi32(v, uvshift), v, mask);
}
static FORCE_INLINE __m256i lookup_AVX2(const int16_t *VXFull, const int16_t *VYFull, const PixelType *pref, int w, const __m256i &dwords_time256, const __m256i &dwords_ref_pitch, const __m256i &dwords_hoffsets) {
__m256i vx = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)&VXFull[w]));
__m256i vy = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)&VYFull[w]));
vx = _mm256_madd_epi16(vx, dwords_time256);
vx = _mm256_srai_epi32(vx, 8);
vy = _mm256_madd_epi16(vy, dwords_time256);
vy = _mm256_srai_epi32(vy, 8);
__m256i addr = _mm256_madd_epi16(vy, dwords_ref_pitch);
addr = _mm256_add_epi32(addr, vx);
addr = _mm256_add_epi32(addr, dwords_hoffsets);
// It's okay to read two or three bytes more than needed. pref is always padded, unless the user chooses a horizontal padding of 0, which would be stupid.
__m256i gathered = _mm256_i32gather_epi32((const int *)pref, addr, sizeof(PixelType));
gathered = _mm256_and_si256(gathered, _mm256_set1_epi32((1 << (sizeof(PixelType) * 8)) - 1));
return gathered;
}
void vpx_highbd_hadamard_32x32_avx2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff) {
int idx;
tran_low_t *t_coeff = coeff;
for (idx = 0; idx < 4; ++idx) {
const int16_t *src_ptr =
src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
vpx_highbd_hadamard_16x16_avx2(src_ptr, src_stride, t_coeff + idx * 256);
}
for (idx = 0; idx < 256; idx += 8) {
__m256i coeff0 = _mm256_loadu_si256((const __m256i *)t_coeff);
__m256i coeff1 = _mm256_loadu_si256((const __m256i *)(t_coeff + 256));
__m256i coeff2 = _mm256_loadu_si256((const __m256i *)(t_coeff + 512));
__m256i coeff3 = _mm256_loadu_si256((const __m256i *)(t_coeff + 768));
__m256i b0 = _mm256_add_epi32(coeff0, coeff1);
__m256i b1 = _mm256_sub_epi32(coeff0, coeff1);
__m256i b2 = _mm256_add_epi32(coeff2, coeff3);
__m256i b3 = _mm256_sub_epi32(coeff2, coeff3);
b0 = _mm256_srai_epi32(b0, 2);
b1 = _mm256_srai_epi32(b1, 2);
b2 = _mm256_srai_epi32(b2, 2);
b3 = _mm256_srai_epi32(b3, 2);
coeff0 = _mm256_add_epi32(b0, b2);
coeff1 = _mm256_add_epi32(b1, b3);
coeff2 = _mm256_sub_epi32(b0, b2);
coeff3 = _mm256_sub_epi32(b1, b3);
_mm256_storeu_si256((__m256i *)coeff, coeff0);
_mm256_storeu_si256((__m256i *)(coeff + 256), coeff1);
_mm256_storeu_si256((__m256i *)(coeff + 512), coeff2);
_mm256_storeu_si256((__m256i *)(coeff + 768), coeff3);
coeff += 8;
t_coeff += 8;
}
}
void cvtScale_s16s32f32Line_AVX2(const short* src, int* dst, float scale, float shift, int width)
{
int x = 0;
__m256 scale256 = _mm256_set1_ps(scale);
__m256 shift256 = _mm256_set1_ps(shift);
const int shuffle = 0xD8;
for (; x <= width - 16; x += 16)
{
__m256i v_src = _mm256_loadu_si256((const __m256i *)(src + x));
v_src = _mm256_permute4x64_epi64(v_src, shuffle);
__m256i v_src_lo = _mm256_srai_epi32(_mm256_unpacklo_epi16(v_src, v_src), 16);
__m256i v_src_hi = _mm256_srai_epi32(_mm256_unpackhi_epi16(v_src, v_src), 16);
__m256 v_dst0 = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(v_src_lo), scale256), shift256);
__m256 v_dst1 = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(v_src_hi), scale256), shift256);
_mm256_storeu_si256((__m256i *)(dst + x), _mm256_cvtps_epi32(v_dst0));
_mm256_storeu_si256((__m256i *)(dst + x + 8), _mm256_cvtps_epi32(v_dst1));
}
for (; x < width; x++)
dst[x] = saturate_cast<int>(src[x] * scale + shift);
}
static INLINE __m256i highbd_comp_mask_pred_line_avx2(const __m256i s0,
const __m256i s1,
const __m256i a) {
const __m256i alpha_max = _mm256_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
const __m256i round_const =
_mm256_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
const __m256i a_inv = _mm256_sub_epi16(alpha_max, a);
const __m256i s_lo = _mm256_unpacklo_epi16(s0, s1);
const __m256i a_lo = _mm256_unpacklo_epi16(a, a_inv);
const __m256i pred_lo = _mm256_madd_epi16(s_lo, a_lo);
const __m256i pred_l = _mm256_srai_epi32(
_mm256_add_epi32(pred_lo, round_const), AOM_BLEND_A64_ROUND_BITS);
const __m256i s_hi = _mm256_unpackhi_epi16(s0, s1);
const __m256i a_hi = _mm256_unpackhi_epi16(a, a_inv);
const __m256i pred_hi = _mm256_madd_epi16(s_hi, a_hi);
const __m256i pred_h = _mm256_srai_epi32(
_mm256_add_epi32(pred_hi, round_const), AOM_BLEND_A64_ROUND_BITS);
const __m256i comp = _mm256_packs_epi32(pred_l, pred_h);
return comp;
}
static FORCE_INLINE void FlowInterSimple_double_8px_AVX2(
int w, PixelType *pdst,
const PixelType *prefB, const PixelType *prefF,
const int16_t *VXFullB, const int16_t *VXFullF,
const int16_t *VYFullB, const int16_t *VYFullF,
const uint8_t *MaskB, const uint8_t *MaskF,
int nPelLog,
const __m256i &dwords_ref_pitch, const __m256i &dwords_hoffsets) {
__m256i dwords_w = _mm256_add_epi32(_mm256_set1_epi32(w << nPelLog), dwords_hoffsets); /// maybe do it another way
__m256i dstF = lookup_double_AVX2(VXFullF, VYFullF, prefF, w, dwords_ref_pitch, dwords_w);
__m256i dstB = lookup_double_AVX2(VXFullB, VYFullB, prefB, w, dwords_ref_pitch, dwords_w);
__m256i maskf = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskF[w]));
__m256i maskb = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskB[w]));
__m256i dstF_dstB = _mm256_add_epi32(dstF, dstB);
dstF_dstB = _mm256_slli_epi32(dstF_dstB, 8);
__m256i dst;
if (sizeof(PixelType) == 1) {
__m256i dstB_dstF = _mm256_sub_epi16(dstB, dstF);
__m256i maskf_maskb = _mm256_sub_epi16(maskf, maskb);
dst = _mm256_madd_epi16(dstB_dstF, maskf_maskb);
} else {
__m256i dstB_dstF = _mm256_sub_epi32(dstB, dstF);
__m256i maskf_maskb = _mm256_sub_epi32(maskf, maskb);
dst = _mm256_mullo_epi32(dstB_dstF, maskf_maskb);
}
dst = _mm256_add_epi32(dst, dstF_dstB);
dst = _mm256_srai_epi32(dst, 9);
dst = _mm256_packus_epi32(dst, dst);
dst = _mm256_permute4x64_epi64(dst, 0xe8); // 0b11101000 - copy third qword to second qword
__m128i dst128 = _mm256_castsi256_si128(dst);
if (sizeof(PixelType) == 1) {
dst128 = _mm_packus_epi16(dst128, dst128);
_mm_storel_epi64((__m128i *)&pdst[w], dst128);
} else {
_mm_storeu_si128((__m128i *)&pdst[w], dst128);
}
}
static INLINE void quantize(const __m256i *qp, __m256i *c,
const int16_t *iscan_ptr, int log_scale,
tran_low_t *qcoeff, tran_low_t *dqcoeff,
__m256i *eob) {
const __m256i abs_coeff = _mm256_abs_epi32(*c);
__m256i q = _mm256_add_epi32(abs_coeff, qp[0]);
__m256i q_lo = _mm256_mul_epi32(q, qp[1]);
__m256i q_hi = _mm256_srli_epi64(q, 32);
const __m256i qp_hi = _mm256_srli_epi64(qp[1], 32);
q_hi = _mm256_mul_epi32(q_hi, qp_hi);
q_lo = _mm256_srli_epi64(q_lo, 16 - log_scale);
q_hi = _mm256_srli_epi64(q_hi, 16 - log_scale);
q_hi = _mm256_slli_epi64(q_hi, 32);
q = _mm256_or_si256(q_lo, q_hi);
const __m256i abs_s = _mm256_slli_epi32(abs_coeff, 1 + log_scale);
const __m256i mask = _mm256_cmpgt_epi32(qp[2], abs_s);
q = _mm256_andnot_si256(mask, q);
__m256i dq = _mm256_mullo_epi32(q, qp[2]);
dq = _mm256_srai_epi32(dq, log_scale);
q = _mm256_sign_epi32(q, *c);
dq = _mm256_sign_epi32(dq, *c);
_mm256_storeu_si256((__m256i *)qcoeff, q);
_mm256_storeu_si256((__m256i *)dqcoeff, dq);
const __m128i isc = _mm_loadu_si128((const __m128i *)iscan_ptr);
const __m128i zr = _mm_setzero_si128();
const __m128i lo = _mm_unpacklo_epi16(isc, zr);
const __m128i hi = _mm_unpackhi_epi16(isc, zr);
const __m256i iscan =
_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1);
const __m256i zero = _mm256_setzero_si256();
const __m256i zc = _mm256_cmpeq_epi32(dq, zero);
const __m256i nz = _mm256_cmpeq_epi32(zc, zero);
__m256i cur_eob = _mm256_sub_epi32(iscan, nz);
cur_eob = _mm256_and_si256(cur_eob, nz);
*eob = _mm256_max_epi32(cur_eob, *eob);
}
int warpAffineBlockline(int *adelta, int *bdelta, short* xy, short* alpha, int X0, int Y0, int bw)
{
const int AB_BITS = MAX(10, (int)INTER_BITS);
int x1 = 0;
__m256i fxy_mask = _mm256_set1_epi32(INTER_TAB_SIZE - 1);
__m256i XX = _mm256_set1_epi32(X0), YY = _mm256_set1_epi32(Y0);
for (; x1 <= bw - 16; x1 += 16)
{
__m256i tx0, tx1, ty0, ty1;
tx0 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(adelta + x1)), XX);
ty0 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(bdelta + x1)), YY);
tx1 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(adelta + x1 + 8)), XX);
ty1 = _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(bdelta + x1 + 8)), YY);
tx0 = _mm256_srai_epi32(tx0, AB_BITS - INTER_BITS);
ty0 = _mm256_srai_epi32(ty0, AB_BITS - INTER_BITS);
tx1 = _mm256_srai_epi32(tx1, AB_BITS - INTER_BITS);
ty1 = _mm256_srai_epi32(ty1, AB_BITS - INTER_BITS);
__m256i fx_ = _mm256_packs_epi32(_mm256_and_si256(tx0, fxy_mask),
_mm256_and_si256(tx1, fxy_mask));
__m256i fy_ = _mm256_packs_epi32(_mm256_and_si256(ty0, fxy_mask),
_mm256_and_si256(ty1, fxy_mask));
tx0 = _mm256_packs_epi32(_mm256_srai_epi32(tx0, INTER_BITS),
_mm256_srai_epi32(tx1, INTER_BITS));
ty0 = _mm256_packs_epi32(_mm256_srai_epi32(ty0, INTER_BITS),
_mm256_srai_epi32(ty1, INTER_BITS));
fx_ = _mm256_adds_epi16(fx_, _mm256_slli_epi16(fy_, INTER_BITS));
fx_ = _mm256_permute4x64_epi64(fx_, (3 << 6) + (1 << 4) + (2 << 2) + 0);
_mm256_storeu_si256((__m256i*)(xy + x1 * 2), _mm256_unpacklo_epi16(tx0, ty0));
_mm256_storeu_si256((__m256i*)(xy + x1 * 2 + 16), _mm256_unpackhi_epi16(tx0, ty0));
_mm256_storeu_si256((__m256i*)(alpha + x1), fx_);
}
_mm256_zeroupper();
return x1;
}
void TransLut_FindIndexAvx2 <TransLut::MapperLog>::find_index (const TransLut::FloatIntMix val_arr [8], __m256i &index, __m256 &frac)
{
assert (val_arr != 0);
// Constants
static const int mant_size = 23;
static const int exp_bias = 127;
static const uint32_t base = (exp_bias + LOGLUT_MIN_L2) << mant_size;
static const float val_min = 1.0f / (int64_t (1) << -LOGLUT_MIN_L2);
// static const float val_max = float (int64_t (1) << LOGLUT_MAX_L2);
static const int frac_size = mant_size - LOGLUT_RES_L2;
static const uint32_t frac_mask = (1 << frac_size) - 1;
const __m256 zero_f = _mm256_setzero_ps ();
const __m256 one_f = _mm256_set1_ps (1);
const __m256 frac_mul = _mm256_set1_ps (1.0f / (1 << frac_size));
const __m256 mul_eps = _mm256_set1_ps (1.0f / val_min);
const __m256 mask_abs_f = _mm256_load_ps (
reinterpret_cast <const float *> (fstb::ToolsAvx2::_mask_abs)
);
const __m256i zero_i = _mm256_setzero_si256 ();
const __m256i mask_abs_epi32 = _mm256_set1_epi32 (0x7FFFFFFF);
const __m256i one_epi32 = _mm256_set1_epi32 (1);
const __m256i base_epi32 = _mm256_set1_epi32 (int (base));
const __m256i frac_mask_epi32 = _mm256_set1_epi32 (frac_mask);
const __m256i val_min_epi32 =
_mm256_set1_epi32 ((LOGLUT_MIN_L2 + exp_bias) << mant_size);
const __m256i val_max_epi32 =
_mm256_set1_epi32 ((LOGLUT_MAX_L2 + exp_bias) << mant_size);
const __m256i index_max_epi32 =
_mm256_set1_epi32 ((LOGLUT_MAX_L2 - LOGLUT_MIN_L2) << LOGLUT_RES_L2);
const __m256i hsize_epi32 = _mm256_set1_epi32 (LOGLUT_HSIZE);
const __m256i mirror_epi32 = _mm256_set1_epi32 (LOGLUT_HSIZE - 1);
// It really starts here
const __m256 val_f = _mm256_load_ps (reinterpret_cast <const float *> (val_arr));
const __m256 val_a = _mm256_and_ps (val_f, mask_abs_f);
const __m256i val_i = _mm256_load_si256 (reinterpret_cast <const __m256i *> (val_arr));
const __m256i val_u = _mm256_and_si256 (val_i, mask_abs_epi32);
// Standard path
__m256i index_std = _mm256_sub_epi32 (val_u, base_epi32);
index_std = _mm256_srli_epi32 (index_std, frac_size);
index_std = _mm256_add_epi32 (index_std, one_epi32);
__m256i frac_stdi = _mm256_and_si256 (val_u, frac_mask_epi32);
__m256 frac_std = _mm256_cvtepi32_ps (frac_stdi);
frac_std = _mm256_mul_ps (frac_std, frac_mul);
// Epsilon path
__m256 frac_eps = _mm256_max_ps (val_a, zero_f);
frac_eps = _mm256_mul_ps (frac_eps, mul_eps);
// Range cases
const __m256i eps_flag_i = _mm256_cmpgt_epi32 (val_min_epi32, val_u);
const __m256i std_flag_i = _mm256_cmpgt_epi32 (val_max_epi32, val_u);
const __m256 eps_flag_f = _mm256_castsi256_ps (eps_flag_i);
const __m256 std_flag_f = _mm256_castsi256_ps (std_flag_i);
__m256i index_tmp =
fstb::ToolsAvx2::select (std_flag_i, index_std, index_max_epi32);
__m256 frac_tmp =
fstb::ToolsAvx2::select (std_flag_f, frac_std, one_f);
index_tmp = fstb::ToolsAvx2::select (eps_flag_i, zero_i, index_tmp);
frac_tmp = fstb::ToolsAvx2::select (eps_flag_f, frac_eps, frac_tmp);
// Sign cases
const __m256i neg_flag_i = _mm256_srai_epi32 (val_i, 31);
const __m256 neg_flag_f = _mm256_castsi256_ps (neg_flag_i);
const __m256i index_neg = _mm256_sub_epi32 (mirror_epi32, index_tmp);
const __m256i index_pos = _mm256_add_epi32 (hsize_epi32, index_tmp);
const __m256 frac_neg = _mm256_sub_ps (one_f, frac_tmp);
index = fstb::ToolsAvx2::select (neg_flag_i, index_neg, index_pos);
frac = fstb::ToolsAvx2::select (neg_flag_f, frac_neg, frac_tmp);
}
static FORCE_INLINE void FlowInterExtra_8px_AVX2(
int w, PixelType *pdst,
const PixelType *prefB, const PixelType *prefF,
const int16_t *VXFullB, const int16_t *VXFullF,
const int16_t *VYFullB, const int16_t *VYFullF,
const uint8_t *MaskB, const uint8_t *MaskF,
int nPelLog,
const int16_t *VXFullBB, const int16_t *VXFullFF,
const int16_t *VYFullBB, const int16_t *VYFullFF,
const __m256i &dwords_time256, const __m256i &dwords_256_time256,
const __m256i &dwords_ref_pitch, const __m256i &dwords_hoffsets) {
__m256i dwords_w = _mm256_add_epi32(_mm256_set1_epi32(w << nPelLog), dwords_hoffsets);
__m256i dstF = lookup_AVX2(VXFullF, VYFullF, prefF, w, dwords_time256, dwords_ref_pitch, dwords_w);
__m256i dstB = lookup_AVX2(VXFullB, VYFullB, prefB, w, dwords_256_time256, dwords_ref_pitch, dwords_w);
__m256i dstFF = lookup_AVX2(VXFullFF, VYFullFF, prefF, w, dwords_time256, dwords_ref_pitch, dwords_w);
__m256i dstBB = lookup_AVX2(VXFullBB, VYFullBB, prefB, w, dwords_256_time256, dwords_ref_pitch, dwords_w);
__m256i minfb = mm256_min_epu<PixelType>(dstF, dstB);
__m256i maxfb = mm256_max_epu<PixelType>(dstF, dstB);
__m256i medianBB = mm256_max_epu<PixelType>(minfb, mm256_min_epu<PixelType>(maxfb, dstBB));
__m256i medianFF = mm256_max_epu<PixelType>(minfb, mm256_min_epu<PixelType>(maxfb, dstFF));
__m256i maskf = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskF[w]));
__m256i maskb = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskB[w]));
const __m256i dwords_255 = _mm256_set1_epi32(255);
__m256i maskf_inv = _mm256_sub_epi32(dwords_255, maskf);
__m256i maskb_inv = _mm256_sub_epi32(dwords_255, maskb);
if (sizeof(PixelType) == 1) {
dstF = _mm256_mullo_epi16(dstF, maskf_inv);
dstB = _mm256_mullo_epi16(dstB, maskb_inv);
medianBB = _mm256_mullo_epi16(medianBB, maskf);
medianFF = _mm256_mullo_epi16(medianFF, maskb);
} else {
dstF = _mm256_mullo_epi32(dstF, maskf_inv);
dstB = _mm256_mullo_epi32(dstB, maskb_inv);
medianBB = _mm256_mullo_epi32(medianBB, maskf);
medianFF = _mm256_mullo_epi32(medianFF, maskb);
}
dstF = _mm256_add_epi32(dstF, dwords_255);
dstB = _mm256_add_epi32(dstB, dwords_255);
dstF = _mm256_add_epi32(dstF, medianBB);
dstB = _mm256_add_epi32(dstB, medianFF);
dstF = _mm256_srai_epi32(dstF, 8);
dstB = _mm256_srai_epi32(dstB, 8);
if (sizeof(PixelType) == 2) {
dstF = _mm256_sub_epi16(dstF, _mm256_set1_epi32(32768));
dstB = _mm256_sub_epi16(dstB, _mm256_set1_epi32(32768));
}
dstF = _mm256_madd_epi16(dstF, dwords_256_time256);
dstB = _mm256_madd_epi16(dstB, dwords_time256);
if (sizeof(PixelType) == 2) {
// dstF = _mm256_add_epi32(dstF, _mm256_slli_epi32(dwords_256_time256, 15));
// dstB = _mm256_add_epi32(dstB, _mm256_slli_epi32(dwords_time256, 15));
// Knowing that they add up to 256, the two additions can be combined.
dstF = _mm256_add_epi32(dstF, _mm256_set1_epi32(256 << 15));
}
__m256i dst = _mm256_add_epi32(dstF, dstB);
dst = _mm256_srai_epi32(dst, 8);
dst = _mm256_packus_epi32(dst, dst);
dst = _mm256_permute4x64_epi64(dst, 0xe8); // 0b11101000 - copy third qword to second qword
__m128i dst128 = _mm256_castsi256_si128(dst);
if (sizeof(PixelType) == 1) {
dst128 = _mm_packus_epi16(dst128, dst128);
_mm_storel_epi64((__m128i *)&pdst[w], dst128);
} else {
_mm_storeu_si128((__m128i *)&pdst[w], dst128);
}
}
static FORCE_INLINE void FlowInter_8px_AVX2(
int w, PixelType *pdst,
const PixelType *prefB, const PixelType *prefF,
const int16_t *VXFullB, const int16_t *VXFullF,
const int16_t *VYFullB, const int16_t *VYFullF,
const uint8_t *MaskB, const uint8_t *MaskF,
int nPelLog,
const __m256i &dwords_time256, const __m256i &dwords_256_time256,
const __m256i &dwords_ref_pitch, const __m256i &dwords_hoffsets) {
__m256i dwords_w = _mm256_add_epi32(_mm256_set1_epi32(w << nPelLog), dwords_hoffsets);
__m256i dstF = lookup_AVX2(VXFullF, VYFullF, prefF, w, dwords_time256, dwords_ref_pitch, dwords_w);
__m256i dstB = lookup_AVX2(VXFullB, VYFullB, prefB, w, dwords_256_time256, dwords_ref_pitch, dwords_w);
__m256i dstF0 = _mm256_i32gather_epi32((const int *)prefF, dwords_w, sizeof(PixelType));
__m256i dstB0 = _mm256_i32gather_epi32((const int *)prefB, dwords_w, sizeof(PixelType));
dstF0 = _mm256_and_si256(dstF0, _mm256_set1_epi32((1 << (sizeof(PixelType) * 8)) - 1));
dstB0 = _mm256_and_si256(dstB0, _mm256_set1_epi32((1 << (sizeof(PixelType) * 8)) - 1));
__m256i maskf = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskF[w]));
__m256i maskb = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskB[w]));
const __m256i dwords_255 = _mm256_set1_epi32(255);
__m256i maskf_inv = _mm256_sub_epi32(dwords_255, maskf);
__m256i maskb_inv = _mm256_sub_epi32(dwords_255, maskb);
__m256i dstF_maskf_inv, dstB_maskb_inv, dstF0_maskb, dstB0_maskf;
if (sizeof(PixelType) == 1) {
dstF_maskf_inv = _mm256_mullo_epi16(dstF, maskf_inv);
dstB_maskb_inv = _mm256_mullo_epi16(dstB, maskb_inv);
dstF0_maskb = _mm256_mullo_epi16(dstF0, maskb);
dstB0_maskf = _mm256_mullo_epi16(dstB0, maskf);
} else {
dstF_maskf_inv = _mm256_mullo_epi32(dstF, maskf_inv);
dstB_maskb_inv = _mm256_mullo_epi32(dstB, maskb_inv);
dstF0_maskb = _mm256_mullo_epi32(dstF0, maskb);
dstB0_maskf = _mm256_mullo_epi32(dstB0, maskf);
}
__m256i f = _mm256_add_epi32(dstF0_maskb, dstB_maskb_inv);
__m256i b = _mm256_add_epi32(dstB0_maskf, dstF_maskf_inv);
if (sizeof(PixelType) == 1) {
f = _mm256_mullo_epi32(f, maskf);
b = _mm256_mullo_epi32(b, maskb);
f = _mm256_add_epi32(f, dwords_255);
b = _mm256_add_epi32(b, dwords_255);
f = _mm256_srai_epi32(f, 8);
b = _mm256_srai_epi32(b, 8);
} else {
const __m256i qwords_255 = _mm256_set1_epi64x(255);
__m256i tempf = _mm256_mul_epu32(f, maskf);
__m256i tempb = _mm256_mul_epu32(b, maskb);
tempf = _mm256_add_epi64(tempf, qwords_255);
tempb = _mm256_add_epi64(tempb, qwords_255);
tempf = _mm256_srli_epi64(tempf, 8);
tempb = _mm256_srli_epi64(tempb, 8);
f = _mm256_srli_epi64(f, 32);
b = _mm256_srli_epi64(b, 32);
f = _mm256_mul_epu32(f, _mm256_srli_epi64(maskf, 32));
b = _mm256_mul_epu32(b, _mm256_srli_epi64(maskb, 32));
f = _mm256_add_epi64(f, qwords_255);
b = _mm256_add_epi64(b, qwords_255);
f = _mm256_srli_epi64(f, 8);
b = _mm256_srli_epi64(b, 8);
f = _mm256_or_si256(tempf, _mm256_slli_epi64(f, 32));
b = _mm256_or_si256(tempb, _mm256_slli_epi64(b, 32));
}
f = _mm256_add_epi32(f, dstF_maskf_inv);
b = _mm256_add_epi32(b, dstB_maskb_inv);
f = _mm256_add_epi32(f, dwords_255);
b = _mm256_add_epi32(b, dwords_255);
f = _mm256_srai_epi32(f, 8);
b = _mm256_srai_epi32(b, 8);
if (sizeof(PixelType) == 1) {
f = _mm256_madd_epi16(f, dwords_256_time256);
b = _mm256_madd_epi16(b, dwords_time256);
} else {
f = _mm256_mullo_epi32(f, dwords_256_time256);
b = _mm256_mullo_epi32(b, dwords_time256);
}
__m256i dst = _mm256_add_epi32(f, b);
dst = _mm256_srai_epi32(dst, 8);
dst = _mm256_packus_epi32(dst, dst);
dst = _mm256_permute4x64_epi64(dst, 0xe8); // 0b11101000 - copy third qword to second qword
__m128i dst128 = _mm256_castsi256_si128(dst);
if (sizeof(PixelType) == 1) {
dst128 = _mm_packus_epi16(dst128, dst128);
_mm_storel_epi64((__m128i *)&pdst[w], dst128);
} else {
_mm_storeu_si128((__m128i *)&pdst[w], dst128);
}
}
/**
* \brief quantize transformed coefficents
*
*/
void kvz_quant_flat_avx2(const encoder_state_t * const state, coeff_t *coef, coeff_t *q_coef, int32_t width,
int32_t height, int8_t type, int8_t scan_idx, int8_t block_type)
{
const encoder_control_t * const encoder = state->encoder_control;
const uint32_t log2_block_size = kvz_g_convert_to_bit[width] + 2;
const uint32_t * const scan = kvz_g_sig_last_scan[scan_idx][log2_block_size - 1];
int32_t qp_scaled = kvz_get_scaled_qp(type, state->global->QP, (encoder->bitdepth - 8) * 6);
const uint32_t log2_tr_size = kvz_g_convert_to_bit[width] + 2;
const int32_t scalinglist_type = (block_type == CU_INTRA ? 0 : 3) + (int8_t)("\0\3\1\2"[type]);
const int32_t *quant_coeff = encoder->scaling_list.quant_coeff[log2_tr_size - 2][scalinglist_type][qp_scaled % 6];
const int32_t transform_shift = MAX_TR_DYNAMIC_RANGE - encoder->bitdepth - log2_tr_size; //!< Represents scaling through forward transform
const int32_t q_bits = QUANT_SHIFT + qp_scaled / 6 + transform_shift;
const int32_t add = ((state->global->slicetype == KVZ_SLICE_I) ? 171 : 85) << (q_bits - 9);
const int32_t q_bits8 = q_bits - 8;
assert(quant_coeff[0] <= (1 << 15) - 1 && quant_coeff[0] >= -(1 << 15)); //Assuming flat values to fit int16_t
uint32_t ac_sum = 0;
__m256i v_ac_sum = _mm256_setzero_si256();
__m256i v_quant_coeff = _mm256_set1_epi16(quant_coeff[0]);
for (int32_t n = 0; n < width * height; n += 16) {
__m256i v_level = _mm256_loadu_si256((__m256i*)&(coef[n]));
__m256i v_sign = _mm256_cmpgt_epi16(_mm256_setzero_si256(), v_level);
v_sign = _mm256_or_si256(v_sign, _mm256_set1_epi16(1));
v_level = _mm256_abs_epi16(v_level);
__m256i low_a = _mm256_unpacklo_epi16(v_level, _mm256_set1_epi16(0));
__m256i high_a = _mm256_unpackhi_epi16(v_level, _mm256_set1_epi16(0));
__m256i low_b = _mm256_unpacklo_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i high_b = _mm256_unpackhi_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i v_level32_a = _mm256_madd_epi16(low_a, low_b);
__m256i v_level32_b = _mm256_madd_epi16(high_a, high_b);
v_level32_a = _mm256_add_epi32(v_level32_a, _mm256_set1_epi32(add));
v_level32_b = _mm256_add_epi32(v_level32_b, _mm256_set1_epi32(add));
v_level32_a = _mm256_srai_epi32(v_level32_a, q_bits);
v_level32_b = _mm256_srai_epi32(v_level32_b, q_bits);
v_level = _mm256_packs_epi32(v_level32_a, v_level32_b);
v_level = _mm256_sign_epi16(v_level, v_sign);
_mm256_storeu_si256((__m256i*)&(q_coef[n]), v_level);
v_ac_sum = _mm256_add_epi32(v_ac_sum, v_level32_a);
v_ac_sum = _mm256_add_epi32(v_ac_sum, v_level32_b);
}
__m128i temp = _mm_add_epi32(_mm256_castsi256_si128(v_ac_sum), _mm256_extracti128_si256(v_ac_sum, 1));
temp = _mm_add_epi32(temp, _mm_shuffle_epi32(temp, KVZ_PERMUTE(2, 3, 0, 1)));
temp = _mm_add_epi32(temp, _mm_shuffle_epi32(temp, KVZ_PERMUTE(1, 0, 1, 0)));
ac_sum += _mm_cvtsi128_si32(temp);
if (!(encoder->sign_hiding && ac_sum >= 2)) return;
int32_t delta_u[LCU_WIDTH*LCU_WIDTH >> 2];
for (int32_t n = 0; n < width * height; n += 16) {
__m256i v_level = _mm256_loadu_si256((__m256i*)&(coef[n]));
v_level = _mm256_abs_epi16(v_level);
__m256i low_a = _mm256_unpacklo_epi16(v_level, _mm256_set1_epi16(0));
__m256i high_a = _mm256_unpackhi_epi16(v_level, _mm256_set1_epi16(0));
__m256i low_b = _mm256_unpacklo_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i high_b = _mm256_unpackhi_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i v_level32_a = _mm256_madd_epi16(low_a, low_b);
__m256i v_level32_b = _mm256_madd_epi16(high_a, high_b);
v_level32_a = _mm256_add_epi32(v_level32_a, _mm256_set1_epi32(add));
v_level32_b = _mm256_add_epi32(v_level32_b, _mm256_set1_epi32(add));
v_level32_a = _mm256_srai_epi32(v_level32_a, q_bits);
v_level32_b = _mm256_srai_epi32(v_level32_b, q_bits);
v_level = _mm256_packs_epi32(v_level32_a, v_level32_b);
__m256i v_coef = _mm256_loadu_si256((__m256i*)&(coef[n]));
__m256i v_coef_a = _mm256_unpacklo_epi16(_mm256_abs_epi16(v_coef), _mm256_set1_epi16(0));
__m256i v_coef_b = _mm256_unpackhi_epi16(_mm256_abs_epi16(v_coef), _mm256_set1_epi16(0));
__m256i v_quant_coeff_a = _mm256_unpacklo_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i v_quant_coeff_b = _mm256_unpackhi_epi16(v_quant_coeff, _mm256_set1_epi16(0));
v_coef_a = _mm256_madd_epi16(v_coef_a, v_quant_coeff_a);
v_coef_b = _mm256_madd_epi16(v_coef_b, v_quant_coeff_b);
v_coef_a = _mm256_sub_epi32(v_coef_a, _mm256_slli_epi32(_mm256_unpacklo_epi16(v_level, _mm256_set1_epi16(0)), q_bits) );
v_coef_b = _mm256_sub_epi32(v_coef_b, _mm256_slli_epi32(_mm256_unpackhi_epi16(v_level, _mm256_set1_epi16(0)), q_bits) );
v_coef_a = _mm256_srai_epi32(v_coef_a, q_bits8);
v_coef_b = _mm256_srai_epi32(v_coef_b, q_bits8);
_mm_storeu_si128((__m128i*)&(delta_u[n+0*4]), _mm256_castsi256_si128(v_coef_a));
_mm_storeu_si128((__m128i*)&(delta_u[n+2*4]), _mm256_extracti128_si256(v_coef_a, 1));
_mm_storeu_si128((__m128i*)&(delta_u[n+1*4]), _mm256_castsi256_si128(v_coef_b));
_mm_storeu_si128((__m128i*)&(delta_u[n+3*4]), _mm256_extracti128_si256(v_coef_b, 1));
}
if (ac_sum >= 2) {
#define SCAN_SET_SIZE 16
#define LOG2_SCAN_SET_SIZE 4
int32_t n, last_cg = -1, abssum = 0, subset, subpos;
for (subset = (width*height - 1) >> LOG2_SCAN_SET_SIZE; subset >= 0; subset--) {
int32_t first_nz_pos_in_cg = SCAN_SET_SIZE, last_nz_pos_in_cg = -1;
subpos = subset << LOG2_SCAN_SET_SIZE;
abssum = 0;
// Find last coeff pos
for (n = SCAN_SET_SIZE - 1; n >= 0; n--) {
if (q_coef[scan[n + subpos]]) {
last_nz_pos_in_cg = n;
break;
}
}
// First coeff pos
for (n = 0; n <SCAN_SET_SIZE; n++) {
if (q_coef[scan[n + subpos]]) {
first_nz_pos_in_cg = n;
break;
}
}
// Sum all kvz_quant coeffs between first and last
for (n = first_nz_pos_in_cg; n <= last_nz_pos_in_cg; n++) {
abssum += q_coef[scan[n + subpos]];
}
if (last_nz_pos_in_cg >= 0 && last_cg == -1) {
last_cg = 1;
}
if (last_nz_pos_in_cg - first_nz_pos_in_cg >= 4) {
int32_t signbit = (q_coef[scan[subpos + first_nz_pos_in_cg]] > 0 ? 0 : 1);
if (signbit != (abssum & 0x1)) { // compare signbit with sum_parity
int32_t min_cost_inc = 0x7fffffff, min_pos = -1, cur_cost = 0x7fffffff;
int16_t final_change = 0, cur_change = 0;
for (n = (last_cg == 1 ? last_nz_pos_in_cg : SCAN_SET_SIZE - 1); n >= 0; n--) {
uint32_t blkPos = scan[n + subpos];
if (q_coef[blkPos] != 0) {
if (delta_u[blkPos] > 0) {
cur_cost = -delta_u[blkPos];
cur_change = 1;
}
else if (n == first_nz_pos_in_cg && abs(q_coef[blkPos]) == 1) {
cur_cost = 0x7fffffff;
}
else {
cur_cost = delta_u[blkPos];
cur_change = -1;
}
}
else if (n < first_nz_pos_in_cg && ((coef[blkPos] >= 0) ? 0 : 1) != signbit) {
cur_cost = 0x7fffffff;
}
else {
cur_cost = -delta_u[blkPos];
cur_change = 1;
}
if (cur_cost < min_cost_inc) {
min_cost_inc = cur_cost;
final_change = cur_change;
min_pos = blkPos;
}
} // CG loop
if (q_coef[min_pos] == 32767 || q_coef[min_pos] == -32768) {
final_change = -1;
}
if (coef[min_pos] >= 0) q_coef[min_pos] += final_change;
else q_coef[min_pos] -= final_change;
} // Hide
}
if (last_cg == 1) last_cg = 0;
}
#undef SCAN_SET_SIZE
#undef LOG2_SCAN_SET_SIZE
}
__m256i test_mm256_srai_epi32(__m256i a) {
// CHECK-LABEL: test_mm256_srai_epi32
// CHECK: call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %{{.*}}, i32 %{{.*}})
return _mm256_srai_epi32(a, 3);
}
__m256i test_mm256_srai_epi32(__m256i a) {
// CHECK: @llvm.x86.avx2.psrai.d
return _mm256_srai_epi32(a, 3);
}
static FORCE_INLINE void FlowInterSimple_generic_8px_AVX2(
int w, PixelType *pdst,
const PixelType *prefB, const PixelType *prefF,
const int16_t *VXFullB, const int16_t *VXFullF,
const int16_t *VYFullB, const int16_t *VYFullF,
const uint8_t *MaskB, const uint8_t *MaskF,
int nPelLog,
const __m256i &dwords_time256, const __m256i &dwords_256_time256,
const __m256i &dwords_ref_pitch, const __m256i &dwords_hoffsets) {
__m256i dwords_w = _mm256_add_epi32(_mm256_set1_epi32(w << nPelLog), dwords_hoffsets);
__m256i dstF = lookup_AVX2(VXFullF, VYFullF, prefF, w, dwords_time256, dwords_ref_pitch, dwords_w);
__m256i dstB = lookup_AVX2(VXFullB, VYFullB, prefB, w, dwords_256_time256, dwords_ref_pitch, dwords_w);
__m256i maskf = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskF[w]));
__m256i maskb = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)&MaskB[w]));
const __m256i dwords_255 = _mm256_set1_epi32(255);
__m256i maskf_inv = _mm256_sub_epi32(dwords_255, maskf);
__m256i maskb_inv = _mm256_sub_epi32(dwords_255, maskb);
__m256i f, b;
if (sizeof(PixelType) == 1) {
__m256i dstF_dstB = _mm256_or_si256(dstF, _mm256_slli_epi32(dstB, 16));
maskf = _mm256_or_si256(_mm256_slli_epi32(maskf, 16), maskf_inv);
maskb = _mm256_or_si256(maskb, _mm256_slli_epi32(maskb_inv, 16));
f = _mm256_madd_epi16(dstF_dstB, maskf);
b = _mm256_madd_epi16(dstF_dstB, maskb);
} else {
__m256i dstF_maskf_inv = _mm256_mullo_epi32(dstF, maskf_inv);
__m256i dstB_maskb_inv = _mm256_mullo_epi32(dstB, maskb_inv);
__m256i dstB_maskf = _mm256_mullo_epi32(dstB, maskf);
__m256i dstF_maskb = _mm256_mullo_epi32(dstF, maskb);
f = _mm256_add_epi32(dstF_maskf_inv, dstB_maskf);
b = _mm256_add_epi32(dstB_maskb_inv, dstF_maskb);
}
f = _mm256_add_epi32(f, dwords_255);
b = _mm256_add_epi32(b, dwords_255);
f = _mm256_srai_epi32(f, 8);
b = _mm256_srai_epi32(b, 8);
if (sizeof(PixelType) == 1) {
f = _mm256_madd_epi16(f, dwords_256_time256);
b = _mm256_madd_epi16(b, dwords_time256);
} else {
f = _mm256_mullo_epi32(f, dwords_256_time256);
b = _mm256_mullo_epi32(b, dwords_time256);
}
__m256i dst = _mm256_add_epi32(f, b);
dst = _mm256_srai_epi32(dst, 8);
dst = _mm256_packus_epi32(dst, dst);
dst = _mm256_permute4x64_epi64(dst, 0xe8); // 0b11101000 - copy third qword to second qword
__m128i dst128 = _mm256_castsi256_si128(dst);
if (sizeof(PixelType) == 1) {
dst128 = _mm_packus_epi16(dst128, dst128);
_mm_storel_epi64((__m128i *)&pdst[w], dst128);
} else {
_mm_storeu_si128((__m128i *)&pdst[w], dst128);
}
}