template <size_t channelCount> void ResizeBilinear(
const uint8_t *src, size_t srcWidth, size_t srcHeight, size_t srcStride,
uint8_t *dst, size_t dstWidth, size_t dstHeight, size_t dstStride)
{
assert(dstWidth >= A);
struct One { uint8_t channels[channelCount]; };
struct Two { uint8_t channels[channelCount*2]; };
size_t size = 2*dstWidth*channelCount;
size_t bufferSize = AlignHi(dstWidth, A)*channelCount*2;
size_t alignedSize = AlignHi(size, DA) - DA;
const size_t step = A*channelCount;
Buffer buffer(bufferSize, dstWidth, dstHeight);
Base::EstimateAlphaIndex(srcHeight, dstHeight, buffer.iy, buffer.ay, 1);
EstimateAlphaIndexX<channelCount>(srcWidth, dstWidth, buffer.ix, buffer.ax);
ptrdiff_t previous = -2;
__m256i a[2];
for(size_t yDst = 0; yDst < dstHeight; yDst++, dst += dstStride)
{
a[0] = _mm256_set1_epi16(int16_t(Base::FRACTION_RANGE - buffer.ay[yDst]));
a[1] = _mm256_set1_epi16(int16_t(buffer.ay[yDst]));
ptrdiff_t sy = buffer.iy[yDst];
int k = 0;
if(sy == previous)
k = 2;
else if(sy == previous + 1)
{
Swap(buffer.bx[0], buffer.bx[1]);
k = 1;
}
previous = sy;
for(; k < 2; k++)
{
Two * pb = (Two *)buffer.bx[k];
const One * psrc = (const One *)(src + (sy + k)*srcStride);
for(size_t x = 0; x < dstWidth; x++)
pb[x] = *(Two *)(psrc + buffer.ix[x]);
uint8_t * pbx = buffer.bx[k];
for(size_t i = 0; i < bufferSize; i += step)
InterpolateX<channelCount>((__m256i*)(buffer.ax + i), (__m256i*)(pbx + i));
}
for(size_t ib = 0, id = 0; ib < alignedSize; ib += DA, id += A)
InterpolateY<true>(buffer.bx[0] + ib, buffer.bx[1] + ib, a, dst + id);
size_t i = size - DA;
InterpolateY<false>(buffer.bx[0] + i, buffer.bx[1] + i, a, dst + i/2);
}
}
int64_t vp9_block_error_avx2(const int16_t *coeff,
const int16_t *dqcoeff,
intptr_t block_size,
int64_t *ssz) {
__m256i sse_reg, ssz_reg, coeff_reg, dqcoeff_reg;
__m256i exp_dqcoeff_lo, exp_dqcoeff_hi, exp_coeff_lo, exp_coeff_hi;
__m256i sse_reg_64hi, ssz_reg_64hi;
__m128i sse_reg128, ssz_reg128;
int64_t sse;
int i;
const __m256i zero_reg = _mm256_set1_epi16(0);
// init sse and ssz registerd to zero
sse_reg = _mm256_set1_epi16(0);
ssz_reg = _mm256_set1_epi16(0);
for (i = 0 ; i < block_size ; i+= 16) {
// load 32 bytes from coeff and dqcoeff
coeff_reg = _mm256_loadu_si256((const __m256i *)(coeff + i));
dqcoeff_reg = _mm256_loadu_si256((const __m256i *)(dqcoeff + i));
// dqcoeff - coeff
dqcoeff_reg = _mm256_sub_epi16(dqcoeff_reg, coeff_reg);
// madd (dqcoeff - coeff)
dqcoeff_reg = _mm256_madd_epi16(dqcoeff_reg, dqcoeff_reg);
// madd coeff
coeff_reg = _mm256_madd_epi16(coeff_reg, coeff_reg);
// expand each double word of madd (dqcoeff - coeff) to quad word
exp_dqcoeff_lo = _mm256_unpacklo_epi32(dqcoeff_reg, zero_reg);
exp_dqcoeff_hi = _mm256_unpackhi_epi32(dqcoeff_reg, zero_reg);
// expand each double word of madd (coeff) to quad word
exp_coeff_lo = _mm256_unpacklo_epi32(coeff_reg, zero_reg);
exp_coeff_hi = _mm256_unpackhi_epi32(coeff_reg, zero_reg);
// add each quad word of madd (dqcoeff - coeff) and madd (coeff)
sse_reg = _mm256_add_epi64(sse_reg, exp_dqcoeff_lo);
ssz_reg = _mm256_add_epi64(ssz_reg, exp_coeff_lo);
sse_reg = _mm256_add_epi64(sse_reg, exp_dqcoeff_hi);
ssz_reg = _mm256_add_epi64(ssz_reg, exp_coeff_hi);
}
// save the higher 64 bit of each 128 bit lane
sse_reg_64hi = _mm256_srli_si256(sse_reg, 8);
ssz_reg_64hi = _mm256_srli_si256(ssz_reg, 8);
// add the higher 64 bit to the low 64 bit
sse_reg = _mm256_add_epi64(sse_reg, sse_reg_64hi);
ssz_reg = _mm256_add_epi64(ssz_reg, ssz_reg_64hi);
// add each 64 bit from each of the 128 bit lane of the 256 bit
sse_reg128 = _mm_add_epi64(_mm256_castsi256_si128(sse_reg),
_mm256_extractf128_si256(sse_reg, 1));
ssz_reg128 = _mm_add_epi64(_mm256_castsi256_si128(ssz_reg),
_mm256_extractf128_si256(ssz_reg, 1));
// store the results
_mm_storel_epi64((__m128i*)(&sse), sse_reg128);
_mm_storel_epi64((__m128i*)(ssz), ssz_reg128);
return sse;
}
static INLINE unsigned int highbd_masked_sad16xh_avx2(
const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
const uint8_t *b8, int b_stride, const uint8_t *m_ptr, int m_stride,
int width, int height) {
const uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src8);
const uint16_t *a_ptr = CONVERT_TO_SHORTPTR(a8);
const uint16_t *b_ptr = CONVERT_TO_SHORTPTR(b8);
int x, y;
__m256i res = _mm256_setzero_si256();
const __m256i mask_max = _mm256_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
const __m256i round_const =
_mm256_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
const __m256i one = _mm256_set1_epi16(1);
for (y = 0; y < height; y++) {
for (x = 0; x < width; x += 16) {
const __m256i src = _mm256_lddqu_si256((const __m256i *)&src_ptr[x]);
const __m256i a = _mm256_lddqu_si256((const __m256i *)&a_ptr[x]);
const __m256i b = _mm256_lddqu_si256((const __m256i *)&b_ptr[x]);
// Zero-extend mask to 16 bits
const __m256i m =
_mm256_cvtepu8_epi16(_mm_lddqu_si128((const __m128i *)&m_ptr[x]));
const __m256i m_inv = _mm256_sub_epi16(mask_max, m);
const __m256i data_l = _mm256_unpacklo_epi16(a, b);
const __m256i mask_l = _mm256_unpacklo_epi16(m, m_inv);
__m256i pred_l = _mm256_madd_epi16(data_l, mask_l);
pred_l = _mm256_srai_epi32(_mm256_add_epi32(pred_l, round_const),
AOM_BLEND_A64_ROUND_BITS);
const __m256i data_r = _mm256_unpackhi_epi16(a, b);
const __m256i mask_r = _mm256_unpackhi_epi16(m, m_inv);
__m256i pred_r = _mm256_madd_epi16(data_r, mask_r);
pred_r = _mm256_srai_epi32(_mm256_add_epi32(pred_r, round_const),
AOM_BLEND_A64_ROUND_BITS);
// Note: the maximum value in pred_l/r is (2^bd)-1 < 2^15,
// so it is safe to do signed saturation here.
const __m256i pred = _mm256_packs_epi32(pred_l, pred_r);
// There is no 16-bit SAD instruction, so we have to synthesize
// an 8-element SAD. We do this by storing 4 32-bit partial SADs,
// and accumulating them at the end
const __m256i diff = _mm256_abs_epi16(_mm256_sub_epi16(pred, src));
res = _mm256_add_epi32(res, _mm256_madd_epi16(diff, one));
}
src_ptr += src_stride;
a_ptr += a_stride;
b_ptr += b_stride;
m_ptr += m_stride;
}
// At this point, we have four 32-bit partial SADs stored in 'res'.
res = _mm256_hadd_epi32(res, res);
res = _mm256_hadd_epi32(res, res);
int sad = _mm256_extract_epi32(res, 0) + _mm256_extract_epi32(res, 4);
return (sad + 31) >> 6;
}
template <bool align> void Bgr48pToBgra32(const uint8_t * blue, size_t blueStride, size_t width, size_t height,
const uint8_t * green, size_t greenStride, const uint8_t * red, size_t redStride, uint8_t * bgra, size_t bgraStride, uint8_t alpha)
{
assert(width >= HA);
if(align)
{
assert(Aligned(blue) && Aligned(blueStride));
assert(Aligned(green) && Aligned(greenStride));
assert(Aligned(red) && Aligned(redStride));
assert(Aligned(bgra) && Aligned(bgraStride));
}
__m256i _alpha = _mm256_slli_si256(_mm256_set1_epi16(alpha), 1);
size_t alignedWidth = AlignLo(width, HA);
for(size_t row = 0; row < height; ++row)
{
for(size_t col = 0, srcOffset = 0, dstOffset = 0; col < alignedWidth; col += HA, srcOffset += A, dstOffset += DA)
Bgr48pToBgra32<align>(bgra + dstOffset, blue, green, red, srcOffset, _alpha);
if(width != alignedWidth)
Bgr48pToBgra32<false>(bgra + (width - HA)*4, blue, green, red, (width - HA)*2, _alpha);
blue += blueStride;
green += greenStride;
red += redStride;
bgra += bgraStride;
}
}
void
maddrc16_imul_avx2(uint8_t* region1, const uint8_t* region2,
uint8_t constant, size_t length)
{
uint8_t *end;
register __m256i reg1, reg2, ri[4], sp[4], mi[4];
const uint8_t *p = pt[constant];
if (constant == 0)
return;
if (constant == 1) {
xorr_avx2(region1, region2, length);
return;
}
mi[0] = _mm256_set1_epi8(0x11);
mi[1] = _mm256_set1_epi8(0x22);
mi[2] = _mm256_set1_epi8(0x44);
mi[3] = _mm256_set1_epi8(0x88);
sp[0] = _mm256_set1_epi16(p[0]);
sp[1] = _mm256_set1_epi16(p[1]);
sp[2] = _mm256_set1_epi16(p[2]);
sp[3] = _mm256_set1_epi16(p[3]);
for (end=region1+length; region1<end; region1+=32, region2+=32) {
reg2 = _mm256_load_si256((void *)region2);
reg1 = _mm256_load_si256((void *)region1);
ri[0] = _mm256_and_si256(reg2, mi[0]);
ri[1] = _mm256_and_si256(reg2, mi[1]);
ri[2] = _mm256_and_si256(reg2, mi[2]);
ri[3] = _mm256_and_si256(reg2, mi[3]);
ri[1] = _mm256_srli_epi16(ri[1], 1);
ri[2] = _mm256_srli_epi16(ri[2], 2);
ri[3] = _mm256_srli_epi16(ri[3], 3);
ri[0] = _mm256_mullo_epi16(ri[0], sp[0]);
ri[1] = _mm256_mullo_epi16(ri[1], sp[1]);
ri[2] = _mm256_mullo_epi16(ri[2], sp[2]);
ri[3] = _mm256_mullo_epi16(ri[3], sp[3]);
ri[0] = _mm256_xor_si256(ri[0], ri[1]);
ri[2] = _mm256_xor_si256(ri[2], ri[3]);
ri[0] = _mm256_xor_si256(ri[0], ri[2]);
ri[0] = _mm256_xor_si256(ri[0], reg1);
_mm256_store_si256((void *)region1, ri[0]);
}
}
static INLINE void hor_transform_row_dual_avx2(__m256i* row){
__m256i mask_pos = _mm256_set1_epi16(1);
__m256i mask_neg = _mm256_set1_epi16(-1);
__m256i sign_mask = _mm256_unpacklo_epi64(mask_pos, mask_neg);
__m256i temp = _mm256_shuffle_epi32(*row, KVZ_PERMUTE(2, 3, 0, 1));
*row = _mm256_sign_epi16(*row, sign_mask);
*row = _mm256_add_epi16(*row, temp);
sign_mask = _mm256_unpacklo_epi32(mask_pos, mask_neg);
temp = _mm256_shuffle_epi32(*row, KVZ_PERMUTE(1, 0, 3, 2));
*row = _mm256_sign_epi16(*row, sign_mask);
*row = _mm256_add_epi16(*row, temp);
sign_mask = _mm256_unpacklo_epi16(mask_pos, mask_neg);
temp = _mm256_shufflelo_epi16(*row, KVZ_PERMUTE(1,0,3,2));
temp = _mm256_shufflehi_epi16(temp, KVZ_PERMUTE(1,0,3,2));
*row = _mm256_sign_epi16(*row, sign_mask);
*row = _mm256_add_epi16(*row, temp);
}
int main() {
const ssize_t A = 3;
const size_t Awidth = 2;
const size_t Dwidth = 4;
const ssize_t Dmin = (-1) * (1ll << (Dwidth - 1));
const ssize_t Dmax = (1ll << (Dwidth - 1)) - 1;
const ssize_t Cwidth = Awidth + Dwidth;
const ssize_t AInv = ext_euklidean(A, Cwidth) & ((1ll << Cwidth) - 1);
const size_t numCodewords = (1ull << Cwidth);
std::cout << "numCodewords: " << numCodewords << std::endl;
const size_t numMasks = numCodewords / (sizeof(int) * 4); // How many masks will we generate?
int * pNonCodewordMasks = new int[numMasks];
const int16_t c = ~((1ll << (Cwidth - 1)) - 1);
std::cout << "c = 0x" << std::hex << c << std::dec << std::endl;
for (ssize_t i = 0, cw = c, posMask = 0; i < numCodewords; ++posMask) {
int tmpMask = 0;
for (ssize_t k = 0; k < 16; ++k, ++cw, ++i) {
if ((cw % A) != 0) { // we want the non-codewords
// std::cout << "cw % A != 0: " << cw << std::endl;
tmpMask |= (1ll << (k * 2)) | (1ll << (k * 2 + 1)); // expand to 32 bits, because AVX2 cannot movemask across lanes to 16 bits
}
}
pNonCodewordMasks[posMask] = tmpMask;
}
std::cout << "numMasks: " << numMasks << std::endl;
std::cout << "non-codeword-masks: 0x" << std::hex << std::setfill('0');
for (size_t posMask = 0; posMask < numMasks; ++posMask) {
std::cout << std::setw(8) << pNonCodewordMasks[posMask] << ':';
}
std::cout << std::dec << std::endl << std::setfill(' ');
auto mmCodewords = _mm256_set_epi16(c+15, c+14, c+13, c+12, c+11, c+10, c+9, c+8, c+7, c+6, c+5, c+4, c+3, c+2, c+1, c);
auto mmAddUp = _mm256_set1_epi16(16);
auto mmAinv = _mm256_set1_epi16(AInv);
auto mmDmin = _mm256_set1_epi16(Dmin);
auto mmDmax = _mm256_set1_epi16(Dmax);
const size_t posEnd = (1ull << Cwidth);
__m256i mmFillUp[] = {_mm256_set1_epi16(0), _mm256_set1_epi16(~((1ll << Cwidth) - 1))}; // fill up all non-codeword bits with 1's if necessary
std::cout << "posEnd = 0x" << std::hex << posEnd << std::dec << std::endl;
std::cout << std::setfill('0') << std::hex;
for(size_t pos = 15, posMask = 0; pos < posEnd; pos += 16, ++posMask) {
auto isNeg = 0x1 & _mm256_movemask_epi8(_mm256_cmpgt_epi16(mmFillUp[0], mmCodewords));
auto mm1 = _mm256_or_si256(_mm256_mullo_epi16(mmCodewords, mmAinv), mmFillUp[isNeg]);
auto mm2 = _mm256_cmpgt_epi16(mm1, mmDmin);
auto mm3 = _mm256_cmpgt_epi16(mmDmax, mm1);
auto mm4 = _mm256_cmpeq_epi16(mmDmax, mm1);
auto mm5 = _mm256_or_si256(mm3, mm4);
auto mm6 = _mm256_and_si256(mm2, mm5);
auto mask = _mm256_movemask_epi8(mm6);
if (mask & pNonCodewordMasks[posMask]) {
std::cout << "BAD @0x" << std::setw((Cwidth + 7) / 8) << pos << ": 0x" << mask << " & 0x" << pNonCodewordMasks[posMask] << " = 0x" << (mask & pNonCodewordMasks[posMask]) << std::endl;
} else {
std::cout << "OK @0x" << std::setw((Cwidth + 7) / 8) << pos << ": 0x" << mask << " & 0x" << pNonCodewordMasks[posMask] << " = 0x" << (mask & pNonCodewordMasks[posMask]) << std::endl;
}
mmCodewords = _mm256_add_epi16(mmCodewords, mmAddUp);
}
std::cout << std::setfill(' ') << std::dec;
}
static INLINE unsigned int masked_sad32xh_avx2(
const uint8_t *src_ptr, int src_stride, const uint8_t *a_ptr, int a_stride,
const uint8_t *b_ptr, int b_stride, const uint8_t *m_ptr, int m_stride,
int width, int height) {
int x, y;
__m256i res = _mm256_setzero_si256();
const __m256i mask_max = _mm256_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
const __m256i round_scale =
_mm256_set1_epi16(1 << (15 - AOM_BLEND_A64_ROUND_BITS));
for (y = 0; y < height; y++) {
for (x = 0; x < width; x += 32) {
const __m256i src = _mm256_lddqu_si256((const __m256i *)&src_ptr[x]);
const __m256i a = _mm256_lddqu_si256((const __m256i *)&a_ptr[x]);
const __m256i b = _mm256_lddqu_si256((const __m256i *)&b_ptr[x]);
const __m256i m = _mm256_lddqu_si256((const __m256i *)&m_ptr[x]);
const __m256i m_inv = _mm256_sub_epi8(mask_max, m);
// Calculate 16 predicted pixels.
// Note that the maximum value of any entry of 'pred_l' or 'pred_r'
// is 64 * 255, so we have plenty of space to add rounding constants.
const __m256i data_l = _mm256_unpacklo_epi8(a, b);
const __m256i mask_l = _mm256_unpacklo_epi8(m, m_inv);
__m256i pred_l = _mm256_maddubs_epi16(data_l, mask_l);
pred_l = _mm256_mulhrs_epi16(pred_l, round_scale);
const __m256i data_r = _mm256_unpackhi_epi8(a, b);
const __m256i mask_r = _mm256_unpackhi_epi8(m, m_inv);
__m256i pred_r = _mm256_maddubs_epi16(data_r, mask_r);
pred_r = _mm256_mulhrs_epi16(pred_r, round_scale);
const __m256i pred = _mm256_packus_epi16(pred_l, pred_r);
res = _mm256_add_epi32(res, _mm256_sad_epu8(pred, src));
}
src_ptr += src_stride;
a_ptr += a_stride;
b_ptr += b_stride;
m_ptr += m_stride;
}
// At this point, we have two 32-bit partial SADs in lanes 0 and 2 of 'res'.
res = _mm256_shuffle_epi32(res, 0xd8);
res = _mm256_permute4x64_epi64(res, 0xd8);
res = _mm256_hadd_epi32(res, res);
res = _mm256_hadd_epi32(res, res);
int32_t sad = _mm256_extract_epi32(res, 0);
return (sad + 31) >> 6;
}
template <bool align> void VectorProduct(const uint8_t * vertical, const uint8_t * horizontal, uint8_t * dst, size_t stride, size_t width, size_t height)
{
assert(width >= A);
if(align)
assert(Aligned(horizontal) && Aligned(dst) && Aligned(stride));
size_t alignedWidth = Simd::AlignLo(width, A);
for(size_t row = 0; row < height; ++row)
{
__m256i _vertical = _mm256_set1_epi16(vertical[row]);
for(size_t col = 0; col < alignedWidth; col += A)
VectorProduct<align>(_vertical, horizontal + col, dst + col);
if(alignedWidth != width)
VectorProduct<false>(_vertical, horizontal + width - A, dst + width - A);
dst += stride;
}
}
static INLINE void variance_kernel_avx2(const __m256i src, const __m256i ref,
__m256i *const sse,
__m256i *const sum) {
const __m256i adj_sub = _mm256_set1_epi16(0xff01); // (1,-1)
// unpack into pairs of source and reference values
const __m256i src_ref0 = _mm256_unpacklo_epi8(src, ref);
const __m256i src_ref1 = _mm256_unpackhi_epi8(src, ref);
// subtract adjacent elements using src*1 + ref*-1
const __m256i diff0 = _mm256_maddubs_epi16(src_ref0, adj_sub);
const __m256i diff1 = _mm256_maddubs_epi16(src_ref1, adj_sub);
const __m256i madd0 = _mm256_madd_epi16(diff0, diff0);
const __m256i madd1 = _mm256_madd_epi16(diff1, diff1);
// add to the running totals
*sum = _mm256_add_epi16(*sum, _mm256_add_epi16(diff0, diff1));
*sse = _mm256_add_epi32(*sse, _mm256_add_epi32(madd0, madd1));
}
static INLINE void comp_mask_pred_line_avx2(const __m256i s0, const __m256i s1,
const __m256i a,
uint8_t *comp_pred) {
const __m256i alpha_max = _mm256_set1_epi8(AOM_BLEND_A64_MAX_ALPHA);
const int16_t round_bits = 15 - AOM_BLEND_A64_ROUND_BITS;
const __m256i round_offset = _mm256_set1_epi16(1 << (round_bits));
const __m256i ma = _mm256_sub_epi8(alpha_max, a);
const __m256i ssAL = _mm256_unpacklo_epi8(s0, s1);
const __m256i aaAL = _mm256_unpacklo_epi8(a, ma);
const __m256i ssAH = _mm256_unpackhi_epi8(s0, s1);
const __m256i aaAH = _mm256_unpackhi_epi8(a, ma);
const __m256i blendAL = _mm256_maddubs_epi16(ssAL, aaAL);
const __m256i blendAH = _mm256_maddubs_epi16(ssAH, aaAH);
const __m256i roundAL = _mm256_mulhrs_epi16(blendAL, round_offset);
const __m256i roundAH = _mm256_mulhrs_epi16(blendAH, round_offset);
const __m256i roundA = _mm256_packus_epi16(roundAL, roundAH);
_mm256_storeu_si256((__m256i *)(comp_pred), roundA);
}
int vpx_satd_avx2(const tran_low_t *coeff, int length) {
const __m256i one = _mm256_set1_epi16(1);
__m256i accum = _mm256_setzero_si256();
int i;
for (i = 0; i < length; i += 16) {
const __m256i src_line = load_tran_low(coeff);
const __m256i abs = _mm256_abs_epi16(src_line);
const __m256i sum = _mm256_madd_epi16(abs, one);
accum = _mm256_add_epi32(accum, sum);
coeff += 16;
}
{ // 32 bit horizontal add
const __m256i a = _mm256_srli_si256(accum, 8);
const __m256i b = _mm256_add_epi32(accum, a);
const __m256i c = _mm256_srli_epi64(b, 32);
const __m256i d = _mm256_add_epi32(b, c);
const __m128i accum_128 = _mm_add_epi32(_mm256_castsi256_si128(d),
_mm256_extractf128_si256(d, 1));
return _mm_cvtsi128_si32(accum_128);
}
}
static INLINE __m256i highbd_comp_mask_pred_line_avx2(const __m256i s0,
const __m256i s1,
const __m256i a) {
const __m256i alpha_max = _mm256_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
const __m256i round_const =
_mm256_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
const __m256i a_inv = _mm256_sub_epi16(alpha_max, a);
const __m256i s_lo = _mm256_unpacklo_epi16(s0, s1);
const __m256i a_lo = _mm256_unpacklo_epi16(a, a_inv);
const __m256i pred_lo = _mm256_madd_epi16(s_lo, a_lo);
const __m256i pred_l = _mm256_srai_epi32(
_mm256_add_epi32(pred_lo, round_const), AOM_BLEND_A64_ROUND_BITS);
const __m256i s_hi = _mm256_unpackhi_epi16(s0, s1);
const __m256i a_hi = _mm256_unpackhi_epi16(a, a_inv);
const __m256i pred_hi = _mm256_madd_epi16(s_hi, a_hi);
const __m256i pred_h = _mm256_srai_epi32(
_mm256_add_epi32(pred_hi, round_const), AOM_BLEND_A64_ROUND_BITS);
const __m256i comp = _mm256_packs_epi32(pred_l, pred_h);
return comp;
}
void av1_build_compound_diffwtd_mask_avx2(uint8_t *mask,
DIFFWTD_MASK_TYPE mask_type,
const uint8_t *src0, int stride0,
const uint8_t *src1, int stride1,
int h, int w) {
const int mb = (mask_type == DIFFWTD_38_INV) ? AOM_BLEND_A64_MAX_ALPHA : 0;
const __m256i y_mask_base = _mm256_set1_epi16(38 - mb);
int i = 0;
if (4 == w) {
do {
const __m128i s0A = xx_loadl_32(src0);
const __m128i s0B = xx_loadl_32(src0 + stride0);
const __m128i s0C = xx_loadl_32(src0 + stride0 * 2);
const __m128i s0D = xx_loadl_32(src0 + stride0 * 3);
const __m128i s0AB = _mm_unpacklo_epi32(s0A, s0B);
const __m128i s0CD = _mm_unpacklo_epi32(s0C, s0D);
const __m128i s0ABCD = _mm_unpacklo_epi64(s0AB, s0CD);
const __m256i s0ABCD_w = _mm256_cvtepu8_epi16(s0ABCD);
const __m128i s1A = xx_loadl_32(src1);
const __m128i s1B = xx_loadl_32(src1 + stride1);
const __m128i s1C = xx_loadl_32(src1 + stride1 * 2);
const __m128i s1D = xx_loadl_32(src1 + stride1 * 3);
const __m128i s1AB = _mm_unpacklo_epi32(s1A, s1B);
const __m128i s1CD = _mm_unpacklo_epi32(s1C, s1D);
const __m128i s1ABCD = _mm_unpacklo_epi64(s1AB, s1CD);
const __m256i s1ABCD_w = _mm256_cvtepu8_epi16(s1ABCD);
const __m256i m16 = calc_mask_avx2(y_mask_base, s0ABCD_w, s1ABCD_w);
const __m256i m8 = _mm256_packus_epi16(m16, _mm256_setzero_si256());
const __m128i x_m8 =
_mm256_castsi256_si128(_mm256_permute4x64_epi64(m8, 0xd8));
xx_storeu_128(mask, x_m8);
src0 += (stride0 << 2);
src1 += (stride1 << 2);
mask += 16;
i += 4;
} while (i < h);
} else if (8 == w) {
do {
const __m128i s0A = xx_loadl_64(src0);
const __m128i s0B = xx_loadl_64(src0 + stride0);
const __m128i s0C = xx_loadl_64(src0 + stride0 * 2);
const __m128i s0D = xx_loadl_64(src0 + stride0 * 3);
const __m256i s0AC_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(s0A, s0C));
const __m256i s0BD_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(s0B, s0D));
const __m128i s1A = xx_loadl_64(src1);
const __m128i s1B = xx_loadl_64(src1 + stride1);
const __m128i s1C = xx_loadl_64(src1 + stride1 * 2);
const __m128i s1D = xx_loadl_64(src1 + stride1 * 3);
const __m256i s1AB_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(s1A, s1C));
const __m256i s1CD_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(s1B, s1D));
const __m256i m16AC = calc_mask_avx2(y_mask_base, s0AC_w, s1AB_w);
const __m256i m16BD = calc_mask_avx2(y_mask_base, s0BD_w, s1CD_w);
const __m256i m8 = _mm256_packus_epi16(m16AC, m16BD);
yy_storeu_256(mask, m8);
src0 += stride0 << 2;
src1 += stride1 << 2;
mask += 32;
i += 4;
} while (i < h);
} else if (16 == w) {
do {
const __m128i s0A = xx_load_128(src0);
const __m128i s0B = xx_load_128(src0 + stride0);
const __m128i s1A = xx_load_128(src1);
const __m128i s1B = xx_load_128(src1 + stride1);
const __m256i s0AL = _mm256_cvtepu8_epi16(s0A);
const __m256i s0BL = _mm256_cvtepu8_epi16(s0B);
const __m256i s1AL = _mm256_cvtepu8_epi16(s1A);
const __m256i s1BL = _mm256_cvtepu8_epi16(s1B);
const __m256i m16AL = calc_mask_avx2(y_mask_base, s0AL, s1AL);
const __m256i m16BL = calc_mask_avx2(y_mask_base, s0BL, s1BL);
const __m256i m8 =
_mm256_permute4x64_epi64(_mm256_packus_epi16(m16AL, m16BL), 0xd8);
yy_storeu_256(mask, m8);
src0 += stride0 << 1;
src1 += stride1 << 1;
mask += 32;
i += 2;
} while (i < h);
} else {
do {
int j = 0;
do {
const __m256i s0 = yy_loadu_256(src0 + j);
const __m256i s1 = yy_loadu_256(src1 + j);
const __m256i s0L = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(s0));
const __m256i s1L = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(s1));
const __m256i s0H =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(s0, 1));
const __m256i s1H =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(s1, 1));
const __m256i m16L = calc_mask_avx2(y_mask_base, s0L, s1L);
const __m256i m16H = calc_mask_avx2(y_mask_base, s0H, s1H);
const __m256i m8 =
_mm256_permute4x64_epi64(_mm256_packus_epi16(m16L, m16H), 0xd8);
yy_storeu_256(mask + j, m8);
j += 32;
} while (j < w);
src0 += stride0;
src1 += stride1;
mask += w;
i += 1;
} while (i < h);
}
}
/**
* \brief quantize transformed coefficents
*
*/
void kvz_quant_flat_avx2(const encoder_state_t * const state, coeff_t *coef, coeff_t *q_coef, int32_t width,
int32_t height, int8_t type, int8_t scan_idx, int8_t block_type)
{
const encoder_control_t * const encoder = state->encoder_control;
const uint32_t log2_block_size = kvz_g_convert_to_bit[width] + 2;
const uint32_t * const scan = kvz_g_sig_last_scan[scan_idx][log2_block_size - 1];
int32_t qp_scaled = kvz_get_scaled_qp(type, state->global->QP, (encoder->bitdepth - 8) * 6);
const uint32_t log2_tr_size = kvz_g_convert_to_bit[width] + 2;
const int32_t scalinglist_type = (block_type == CU_INTRA ? 0 : 3) + (int8_t)("\0\3\1\2"[type]);
const int32_t *quant_coeff = encoder->scaling_list.quant_coeff[log2_tr_size - 2][scalinglist_type][qp_scaled % 6];
const int32_t transform_shift = MAX_TR_DYNAMIC_RANGE - encoder->bitdepth - log2_tr_size; //!< Represents scaling through forward transform
const int32_t q_bits = QUANT_SHIFT + qp_scaled / 6 + transform_shift;
const int32_t add = ((state->global->slicetype == KVZ_SLICE_I) ? 171 : 85) << (q_bits - 9);
const int32_t q_bits8 = q_bits - 8;
assert(quant_coeff[0] <= (1 << 15) - 1 && quant_coeff[0] >= -(1 << 15)); //Assuming flat values to fit int16_t
uint32_t ac_sum = 0;
__m256i v_ac_sum = _mm256_setzero_si256();
__m256i v_quant_coeff = _mm256_set1_epi16(quant_coeff[0]);
for (int32_t n = 0; n < width * height; n += 16) {
__m256i v_level = _mm256_loadu_si256((__m256i*)&(coef[n]));
__m256i v_sign = _mm256_cmpgt_epi16(_mm256_setzero_si256(), v_level);
v_sign = _mm256_or_si256(v_sign, _mm256_set1_epi16(1));
v_level = _mm256_abs_epi16(v_level);
__m256i low_a = _mm256_unpacklo_epi16(v_level, _mm256_set1_epi16(0));
__m256i high_a = _mm256_unpackhi_epi16(v_level, _mm256_set1_epi16(0));
__m256i low_b = _mm256_unpacklo_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i high_b = _mm256_unpackhi_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i v_level32_a = _mm256_madd_epi16(low_a, low_b);
__m256i v_level32_b = _mm256_madd_epi16(high_a, high_b);
v_level32_a = _mm256_add_epi32(v_level32_a, _mm256_set1_epi32(add));
v_level32_b = _mm256_add_epi32(v_level32_b, _mm256_set1_epi32(add));
v_level32_a = _mm256_srai_epi32(v_level32_a, q_bits);
v_level32_b = _mm256_srai_epi32(v_level32_b, q_bits);
v_level = _mm256_packs_epi32(v_level32_a, v_level32_b);
v_level = _mm256_sign_epi16(v_level, v_sign);
_mm256_storeu_si256((__m256i*)&(q_coef[n]), v_level);
v_ac_sum = _mm256_add_epi32(v_ac_sum, v_level32_a);
v_ac_sum = _mm256_add_epi32(v_ac_sum, v_level32_b);
}
__m128i temp = _mm_add_epi32(_mm256_castsi256_si128(v_ac_sum), _mm256_extracti128_si256(v_ac_sum, 1));
temp = _mm_add_epi32(temp, _mm_shuffle_epi32(temp, KVZ_PERMUTE(2, 3, 0, 1)));
temp = _mm_add_epi32(temp, _mm_shuffle_epi32(temp, KVZ_PERMUTE(1, 0, 1, 0)));
ac_sum += _mm_cvtsi128_si32(temp);
if (!(encoder->sign_hiding && ac_sum >= 2)) return;
int32_t delta_u[LCU_WIDTH*LCU_WIDTH >> 2];
for (int32_t n = 0; n < width * height; n += 16) {
__m256i v_level = _mm256_loadu_si256((__m256i*)&(coef[n]));
v_level = _mm256_abs_epi16(v_level);
__m256i low_a = _mm256_unpacklo_epi16(v_level, _mm256_set1_epi16(0));
__m256i high_a = _mm256_unpackhi_epi16(v_level, _mm256_set1_epi16(0));
__m256i low_b = _mm256_unpacklo_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i high_b = _mm256_unpackhi_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i v_level32_a = _mm256_madd_epi16(low_a, low_b);
__m256i v_level32_b = _mm256_madd_epi16(high_a, high_b);
v_level32_a = _mm256_add_epi32(v_level32_a, _mm256_set1_epi32(add));
v_level32_b = _mm256_add_epi32(v_level32_b, _mm256_set1_epi32(add));
v_level32_a = _mm256_srai_epi32(v_level32_a, q_bits);
v_level32_b = _mm256_srai_epi32(v_level32_b, q_bits);
v_level = _mm256_packs_epi32(v_level32_a, v_level32_b);
__m256i v_coef = _mm256_loadu_si256((__m256i*)&(coef[n]));
__m256i v_coef_a = _mm256_unpacklo_epi16(_mm256_abs_epi16(v_coef), _mm256_set1_epi16(0));
__m256i v_coef_b = _mm256_unpackhi_epi16(_mm256_abs_epi16(v_coef), _mm256_set1_epi16(0));
__m256i v_quant_coeff_a = _mm256_unpacklo_epi16(v_quant_coeff, _mm256_set1_epi16(0));
__m256i v_quant_coeff_b = _mm256_unpackhi_epi16(v_quant_coeff, _mm256_set1_epi16(0));
v_coef_a = _mm256_madd_epi16(v_coef_a, v_quant_coeff_a);
v_coef_b = _mm256_madd_epi16(v_coef_b, v_quant_coeff_b);
v_coef_a = _mm256_sub_epi32(v_coef_a, _mm256_slli_epi32(_mm256_unpacklo_epi16(v_level, _mm256_set1_epi16(0)), q_bits) );
v_coef_b = _mm256_sub_epi32(v_coef_b, _mm256_slli_epi32(_mm256_unpackhi_epi16(v_level, _mm256_set1_epi16(0)), q_bits) );
v_coef_a = _mm256_srai_epi32(v_coef_a, q_bits8);
v_coef_b = _mm256_srai_epi32(v_coef_b, q_bits8);
_mm_storeu_si128((__m128i*)&(delta_u[n+0*4]), _mm256_castsi256_si128(v_coef_a));
_mm_storeu_si128((__m128i*)&(delta_u[n+2*4]), _mm256_extracti128_si256(v_coef_a, 1));
_mm_storeu_si128((__m128i*)&(delta_u[n+1*4]), _mm256_castsi256_si128(v_coef_b));
_mm_storeu_si128((__m128i*)&(delta_u[n+3*4]), _mm256_extracti128_si256(v_coef_b, 1));
}
if (ac_sum >= 2) {
#define SCAN_SET_SIZE 16
#define LOG2_SCAN_SET_SIZE 4
int32_t n, last_cg = -1, abssum = 0, subset, subpos;
for (subset = (width*height - 1) >> LOG2_SCAN_SET_SIZE; subset >= 0; subset--) {
int32_t first_nz_pos_in_cg = SCAN_SET_SIZE, last_nz_pos_in_cg = -1;
subpos = subset << LOG2_SCAN_SET_SIZE;
abssum = 0;
// Find last coeff pos
for (n = SCAN_SET_SIZE - 1; n >= 0; n--) {
if (q_coef[scan[n + subpos]]) {
last_nz_pos_in_cg = n;
break;
}
}
// First coeff pos
for (n = 0; n <SCAN_SET_SIZE; n++) {
if (q_coef[scan[n + subpos]]) {
first_nz_pos_in_cg = n;
break;
}
}
// Sum all kvz_quant coeffs between first and last
for (n = first_nz_pos_in_cg; n <= last_nz_pos_in_cg; n++) {
abssum += q_coef[scan[n + subpos]];
}
if (last_nz_pos_in_cg >= 0 && last_cg == -1) {
last_cg = 1;
}
if (last_nz_pos_in_cg - first_nz_pos_in_cg >= 4) {
int32_t signbit = (q_coef[scan[subpos + first_nz_pos_in_cg]] > 0 ? 0 : 1);
if (signbit != (abssum & 0x1)) { // compare signbit with sum_parity
int32_t min_cost_inc = 0x7fffffff, min_pos = -1, cur_cost = 0x7fffffff;
int16_t final_change = 0, cur_change = 0;
for (n = (last_cg == 1 ? last_nz_pos_in_cg : SCAN_SET_SIZE - 1); n >= 0; n--) {
uint32_t blkPos = scan[n + subpos];
if (q_coef[blkPos] != 0) {
if (delta_u[blkPos] > 0) {
cur_cost = -delta_u[blkPos];
cur_change = 1;
}
else if (n == first_nz_pos_in_cg && abs(q_coef[blkPos]) == 1) {
cur_cost = 0x7fffffff;
}
else {
cur_cost = delta_u[blkPos];
cur_change = -1;
}
}
else if (n < first_nz_pos_in_cg && ((coef[blkPos] >= 0) ? 0 : 1) != signbit) {
cur_cost = 0x7fffffff;
}
else {
cur_cost = -delta_u[blkPos];
cur_change = 1;
}
if (cur_cost < min_cost_inc) {
min_cost_inc = cur_cost;
final_change = cur_change;
min_pos = blkPos;
}
} // CG loop
if (q_coef[min_pos] == 32767 || q_coef[min_pos] == -32768) {
final_change = -1;
}
if (coef[min_pos] >= 0) q_coef[min_pos] += final_change;
else q_coef[min_pos] -= final_change;
} // Hide
}
if (last_cg == 1) last_cg = 0;
}
#undef SCAN_SET_SIZE
#undef LOG2_SCAN_SET_SIZE
}
const int8_t POS_LIMIT = INT8_MAX - matrix->max - 1; int8_t score = NEG_LIMIT; int8_t matches = NEG_LIMIT; int8_t similar = NEG_LIMIT; int8_t length = NEG_LIMIT; __m256i vNegLimit = _mm256_set1_epi8(NEG_LIMIT); __m256i vPosLimit = _mm256_set1_epi8(POS_LIMIT); __m256i vSaturationCheckMin = vPosLimit; __m256i vSaturationCheckMax = vNegLimit; __m256i vNegInf = _mm256_set1_epi8(NEG_LIMIT); __m256i vNegInf0 = _mm256_srli_si256_rpl(vNegInf, 1); /* shift in a 0 */ __m256i vOpen = _mm256_set1_epi8(open); __m256i vGap = _mm256_set1_epi8(gap); __m256i vZero = _mm256_set1_epi8(0); __m256i vOne = _mm256_set1_epi8(1); __m256i vOne16 = _mm256_set1_epi16(1); __m256i vNegOne16 = _mm256_set1_epi16(-1); __m256i vN16 = _mm256_set1_epi16(N); __m256i vILo16 = _mm256_set_epi16(16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31); __m256i vIHi16 = _mm256_set_epi16(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15); __m256i vJresetLo16 = _mm256_set_epi16(-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,-26,-27,-28,-29,-30,-31); __m256i vJresetHi16 = _mm256_set_epi16(0,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15); __m256i vMaxH = vNegInf; __m256i vMaxM = vNegInf; __m256i vMaxS = vNegInf; __m256i vMaxL = vNegInf; __m256i vEndILo = vNegInf; __m256i vEndIHi = vNegInf; __m256i vEndJLo = vNegInf; __m256i vEndJHi = vNegInf; __m256i vILimit16 = _mm256_set1_epi16(s1Len);
AVG_NEXT_SRC(src_reg, 1)
// average between previous average to current average
src_avg = _mm256_avg_epu8(src_avg, src_reg);
// expand each byte to 2 bytes
MERGE_WITH_SRC(src_avg, zero_reg)
// save current source average
CALC_SUM_SSE_INSIDE_LOOP
dst+= dst_stride;
}
// x_offset = 8 and y_offset = bilin interpolation
} else {
__m256i filter, pw8, src_next_reg, src_avg;
y_offset <<= 5;
filter = _mm256_load_si256((__m256i const *)
(bilinear_filters_avx2 + y_offset));
pw8 = _mm256_set1_epi16(8);
// load source and another source starting from the next
// following byte
src_reg = _mm256_loadu_si256((__m256i const *) (src));
AVG_NEXT_SRC(src_reg, 1)
for (i = 0; i < height ; i++) {
// save current source average
src_avg = src_reg;
src+= src_stride;
LOAD_SRC_DST
AVG_NEXT_SRC(src_reg, 1)
MERGE_WITH_SRC(src_avg, src_reg)
FILTER_SRC(filter)
CALC_SUM_SSE_INSIDE_LOOP
dst+= dst_stride;
}
void vec_i8_cnt_dosage2(const int8_t *p, int8_t *out, size_t n, int8_t val,
int8_t missing, int8_t missing_substitute)
{
#ifdef COREARRAY_SIMD_SSE2
// header 1, 16-byte aligned
size_t h = (16 - ((size_t)out & 0x0F)) & 0x0F;
for (; (n > 0) && (h > 0); n--, h--, p+=2)
{
*out ++ = ((p[0] == missing) || (p[1] == missing)) ?
missing_substitute :
(p[0]==val ? 1 : 0) + (p[1]==val ? 1 : 0);
}
// body, SSE2
const __m128i val16 = _mm_set1_epi8(val);
const __m128i miss16 = _mm_set1_epi8(missing);
const __m128i sub16 = _mm_set1_epi8(missing_substitute);
const __m128i mask = _mm_set1_epi16(0x00FF);
# ifdef COREARRAY_SIMD_AVX2
// header 2, 32-byte aligned
if ((n >= 16) && ((size_t)out & 0x10))
{
__m128i w1 = MM_LOADU_128((__m128i const*)p); p += 16;
__m128i w2 = MM_LOADU_128((__m128i const*)p); p += 16;
__m128i v1 = _mm_packus_epi16(_mm_and_si128(w1, mask), _mm_and_si128(w2, mask));
__m128i v2 = _mm_packus_epi16(_mm_srli_epi16(w1, 8), _mm_srli_epi16(w2, 8));
__m128i c = _mm_setzero_si128();
c = _mm_sub_epi8(c, _mm_cmpeq_epi8(v1, val16));
c = _mm_sub_epi8(c, _mm_cmpeq_epi8(v2, val16));
w1 = _mm_cmpeq_epi8(v1, miss16);
w2 = _mm_cmpeq_epi8(v2, miss16);
__m128i w = _mm_or_si128(w1, w2);
c = _mm_or_si128(_mm_and_si128(w, sub16), _mm_andnot_si128(w, c));
_mm_store_si128((__m128i *)out, c);
n -= 16; out += 16;
}
const __m256i val32 = _mm256_set1_epi8(val);
const __m256i miss32 = _mm256_set1_epi8(missing);
const __m256i sub32 = _mm256_set1_epi8(missing_substitute);
const __m256i mask2 = _mm256_set1_epi16(0x00FF);
for (; n >= 32; n-=32)
{
__m256i w1 = MM_LOADU_256((__m256i const*)p); p += 32;
__m256i w2 = MM_LOADU_256((__m256i const*)p); p += 32;
__m256i v1 = _mm256_packus_epi16(_mm256_and_si256(w1, mask2), _mm256_and_si256(w2, mask2));
__m256i v2 = _mm256_packus_epi16(_mm256_srli_epi16(w1, 8), _mm256_srli_epi16(w2, 8));
__m256i c = _mm256_setzero_si256();
c = _mm256_sub_epi8(c, _mm256_cmpeq_epi8(v1, val32));
c = _mm256_sub_epi8(c, _mm256_cmpeq_epi8(v2, val32));
w1 = _mm256_cmpeq_epi8(v1, miss32);
w2 = _mm256_cmpeq_epi8(v2, miss32);
__m256i w = _mm256_or_si256(w1, w2);
c = _mm256_or_si256(_mm256_and_si256(w, sub32), _mm256_andnot_si256(w, c));
c = _mm256_permute4x64_epi64(c, 0xD8);
_mm256_store_si256((__m256i *)out, c);
out += 32;
}
# endif
// SSE2 only
for (; n >= 16; n-=16)
{
__m128i w1 = MM_LOADU_128((__m128i const*)p); p += 16;
__m128i w2 = MM_LOADU_128((__m128i const*)p); p += 16;
__m128i v1 = _mm_packus_epi16(_mm_and_si128(w1, mask), _mm_and_si128(w2, mask));
__m128i v2 = _mm_packus_epi16(_mm_srli_epi16(w1, 8), _mm_srli_epi16(w2, 8));
__m128i c = _mm_setzero_si128();
c = _mm_sub_epi8(c, _mm_cmpeq_epi8(v1, val16));
c = _mm_sub_epi8(c, _mm_cmpeq_epi8(v2, val16));
w1 = _mm_cmpeq_epi8(v1, miss16);
w2 = _mm_cmpeq_epi8(v2, miss16);
__m128i w = _mm_or_si128(w1, w2);
c = _mm_or_si128(_mm_and_si128(w, sub16), _mm_andnot_si128(w, c));
_mm_store_si128((__m128i *)out, c);
out += 16;
}
#endif
// tail
for (; n > 0; n--, p+=2)
{
*out ++ = ((p[0] == missing) || (p[1] == missing)) ?
missing_substitute :
(p[0]==val ? 1 : 0) + (p[1]==val ? 1 : 0);
}
}
unsigned int vp9_sub_pixel_variance32xh_avx2(const uint8_t *src,
int src_stride,
int x_offset,
int y_offset,
const uint8_t *dst,
int dst_stride,
int height,
unsigned int *sse) {
__m256i src_reg, dst_reg, exp_src_lo, exp_src_hi, exp_dst_lo, exp_dst_hi;
__m256i sse_reg, sum_reg, sse_reg_hi, res_cmp, sum_reg_lo, sum_reg_hi;
__m256i zero_reg;
int i, sum;
sum_reg = _mm256_set1_epi16(0);
sse_reg = _mm256_set1_epi16(0);
zero_reg = _mm256_set1_epi16(0);
// x_offset = 0 and y_offset = 0
if (x_offset == 0) {
if (y_offset == 0) {
for (i = 0; i < height ; i++) {
LOAD_SRC_DST
// expend each byte to 2 bytes
MERGE_WITH_SRC(src_reg, zero_reg)
CALC_SUM_SSE_INSIDE_LOOP
src+= src_stride;
dst+= dst_stride;
}
// x_offset = 0 and y_offset = 8
} else if (y_offset == 8) {
__m256i src_next_reg;
for (i = 0; i < height ; i++) {
LOAD_SRC_DST
AVG_NEXT_SRC(src_reg, src_stride)
// expend each byte to 2 bytes
MERGE_WITH_SRC(src_reg, zero_reg)
CALC_SUM_SSE_INSIDE_LOOP
src+= src_stride;
dst+= dst_stride;
}
// x_offset = 0 and y_offset = bilin interpolation
} else {
__m256i filter, pw8, src_next_reg;
y_offset <<= 5;
filter = _mm256_load_si256((__m256i const *)
(bilinear_filters_avx2 + y_offset));
pw8 = _mm256_set1_epi16(8);
for (i = 0; i < height ; i++) {
LOAD_SRC_DST
MERGE_NEXT_SRC(src_reg, src_stride)
FILTER_SRC(filter)
CALC_SUM_SSE_INSIDE_LOOP
src+= src_stride;
dst+= dst_stride;
}
}
// x_offset = 8 and y_offset = 0
} else if (x_offset == 8) {
if (y_offset == 0) {
__m256i src_next_reg;
for (i = 0; i < height ; i++) {
LOAD_SRC_DST
AVG_NEXT_SRC(src_reg, 1)
// expand each byte to 2 bytes
MERGE_WITH_SRC(src_reg, zero_reg)
CALC_SUM_SSE_INSIDE_LOOP
src+= src_stride;
dst+= dst_stride;
}
// x_offset = 8 and y_offset = 8
} else if (y_offset == 8) {
__m256i src_next_reg, src_avg;
// load source and another source starting from the next
// following byte
src_reg = _mm256_loadu_si256((__m256i const *) (src));
AVG_NEXT_SRC(src_reg, 1)
for (i = 0; i < height ; i++) {
src_avg = src_reg;
src+= src_stride;
LOAD_SRC_DST
AVG_NEXT_SRC(src_reg, 1)
// average between previous average to current average
src_avg = _mm256_avg_epu8(src_avg, src_reg);
// expand each byte to 2 bytes
MERGE_WITH_SRC(src_avg, zero_reg)
// save current source average
CALC_SUM_SSE_INSIDE_LOOP
dst+= dst_stride;
}
// x_offset = 8 and y_offset = bilin interpolation
} else {
static void vpx_filter_block1d16_v8_avx2(const uint8_t *src_ptr,
ptrdiff_t src_pitch,
uint8_t *output_ptr,
ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m256i addFilterReg64;
__m256i srcReg32b1, srcReg32b2, srcReg32b3, srcReg32b4, srcReg32b5;
__m256i srcReg32b6, srcReg32b7, srcReg32b8, srcReg32b9, srcReg32b10;
__m256i srcReg32b11, srcReg32b12, filtersReg32;
__m256i firstFilters, secondFilters, thirdFilters, forthFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
// create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
addFilterReg64 = _mm256_set1_epi32((int)0x0400040u);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg =_mm_packs_epi16(filtersReg, filtersReg);
// have the same data in both lanes of a 256 bit register
filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
// duplicate only the first 16 bits (first and second byte)
// across 256 bit register
firstFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x100u));
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x504u));
// duplicate only the forth 16 bits (seventh and eighth byte)
// across 256 bit register
forthFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x706u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
// load 16 bytes 7 times in stride of src_pitch
srcReg32b1 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr)));
srcReg32b2 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch)));
srcReg32b3 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)));
srcReg32b4 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)));
srcReg32b5 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)));
srcReg32b6 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)));
srcReg32b7 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)));
// have each consecutive loads on the same 256 register
srcReg32b1 = _mm256_inserti128_si256(srcReg32b1,
_mm256_castsi256_si128(srcReg32b2), 1);
srcReg32b2 = _mm256_inserti128_si256(srcReg32b2,
_mm256_castsi256_si128(srcReg32b3), 1);
srcReg32b3 = _mm256_inserti128_si256(srcReg32b3,
_mm256_castsi256_si128(srcReg32b4), 1);
srcReg32b4 = _mm256_inserti128_si256(srcReg32b4,
_mm256_castsi256_si128(srcReg32b5), 1);
srcReg32b5 = _mm256_inserti128_si256(srcReg32b5,
_mm256_castsi256_si128(srcReg32b6), 1);
srcReg32b6 = _mm256_inserti128_si256(srcReg32b6,
_mm256_castsi256_si128(srcReg32b7), 1);
// merge every two consecutive registers except the last one
srcReg32b10 = _mm256_unpacklo_epi8(srcReg32b1, srcReg32b2);
srcReg32b1 = _mm256_unpackhi_epi8(srcReg32b1, srcReg32b2);
// save
srcReg32b11 = _mm256_unpacklo_epi8(srcReg32b3, srcReg32b4);
// save
srcReg32b3 = _mm256_unpackhi_epi8(srcReg32b3, srcReg32b4);
// save
srcReg32b2 = _mm256_unpacklo_epi8(srcReg32b5, srcReg32b6);
// save
srcReg32b5 = _mm256_unpackhi_epi8(srcReg32b5, srcReg32b6);
for (i = output_height; i > 1; i-=2) {
// load the last 2 loads of 16 bytes and have every two
// consecutive loads in the same 256 bit register
srcReg32b8 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7)));
srcReg32b7 = _mm256_inserti128_si256(srcReg32b7,
_mm256_castsi256_si128(srcReg32b8), 1);
srcReg32b9 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 8)));
srcReg32b8 = _mm256_inserti128_si256(srcReg32b8,
_mm256_castsi256_si128(srcReg32b9), 1);
// merge every two consecutive registers
// save
srcReg32b4 = _mm256_unpacklo_epi8(srcReg32b7, srcReg32b8);
srcReg32b7 = _mm256_unpackhi_epi8(srcReg32b7, srcReg32b8);
// multiply 2 adjacent elements with the filter and add the result
srcReg32b10 = _mm256_maddubs_epi16(srcReg32b10, firstFilters);
srcReg32b6 = _mm256_maddubs_epi16(srcReg32b4, forthFilters);
// add and saturate the results together
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, srcReg32b6);
// multiply 2 adjacent elements with the filter and add the result
srcReg32b8 = _mm256_maddubs_epi16(srcReg32b11, secondFilters);
srcReg32b12 = _mm256_maddubs_epi16(srcReg32b2, thirdFilters);
// add and saturate the results together
srcReg32b10 = _mm256_adds_epi16(srcReg32b10,
_mm256_min_epi16(srcReg32b8, srcReg32b12));
srcReg32b10 = _mm256_adds_epi16(srcReg32b10,
_mm256_max_epi16(srcReg32b8, srcReg32b12));
// multiply 2 adjacent elements with the filter and add the result
srcReg32b1 = _mm256_maddubs_epi16(srcReg32b1, firstFilters);
srcReg32b6 = _mm256_maddubs_epi16(srcReg32b7, forthFilters);
srcReg32b1 = _mm256_adds_epi16(srcReg32b1, srcReg32b6);
// multiply 2 adjacent elements with the filter and add the result
srcReg32b8 = _mm256_maddubs_epi16(srcReg32b3, secondFilters);
srcReg32b12 = _mm256_maddubs_epi16(srcReg32b5, thirdFilters);
// add and saturate the results together
srcReg32b1 = _mm256_adds_epi16(srcReg32b1,
_mm256_min_epi16(srcReg32b8, srcReg32b12));
srcReg32b1 = _mm256_adds_epi16(srcReg32b1,
_mm256_max_epi16(srcReg32b8, srcReg32b12));
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, addFilterReg64);
srcReg32b1 = _mm256_adds_epi16(srcReg32b1, addFilterReg64);
// shift by 7 bit each 16 bit
srcReg32b10 = _mm256_srai_epi16(srcReg32b10, 7);
srcReg32b1 = _mm256_srai_epi16(srcReg32b1, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcReg32b1 = _mm256_packus_epi16(srcReg32b10, srcReg32b1);
src_ptr+=src_stride;
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr,
_mm256_castsi256_si128(srcReg32b1));
// save the next 16 bits
_mm_store_si128((__m128i*)(output_ptr+out_pitch),
_mm256_extractf128_si256(srcReg32b1, 1));
output_ptr+=dst_stride;
// save part of the registers for next strides
srcReg32b10 = srcReg32b11;
srcReg32b1 = srcReg32b3;
srcReg32b11 = srcReg32b2;
srcReg32b3 = srcReg32b5;
srcReg32b2 = srcReg32b4;
srcReg32b5 = srcReg32b7;
srcReg32b7 = srcReg32b9;
}
if (i > 0) {
__m128i srcRegFilt1, srcRegFilt3, srcRegFilt4, srcRegFilt5;
__m128i srcRegFilt6, srcRegFilt7, srcRegFilt8;
// load the last 16 bytes
srcRegFilt8 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7));
// merge the last 2 results together
srcRegFilt4 = _mm_unpacklo_epi8(
_mm256_castsi256_si128(srcReg32b7), srcRegFilt8);
srcRegFilt7 = _mm_unpackhi_epi8(
_mm256_castsi256_si128(srcReg32b7), srcRegFilt8);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt1 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b10),
_mm256_castsi256_si128(firstFilters));
srcRegFilt4 = _mm_maddubs_epi16(srcRegFilt4,
_mm256_castsi256_si128(forthFilters));
srcRegFilt3 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b1),
_mm256_castsi256_si128(firstFilters));
srcRegFilt7 = _mm_maddubs_epi16(srcRegFilt7,
_mm256_castsi256_si128(forthFilters));
// add and saturate the results together
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt4);
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3, srcRegFilt7);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt4 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b11),
_mm256_castsi256_si128(secondFilters));
srcRegFilt5 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b3),
_mm256_castsi256_si128(secondFilters));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt6 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b2),
_mm256_castsi256_si128(thirdFilters));
srcRegFilt7 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b5),
_mm256_castsi256_si128(thirdFilters));
// add and saturate the results together
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1,
_mm_min_epi16(srcRegFilt4, srcRegFilt6));
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3,
_mm_min_epi16(srcRegFilt5, srcRegFilt7));
// add and saturate the results together
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1,
_mm_max_epi16(srcRegFilt4, srcRegFilt6));
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3,
_mm_max_epi16(srcRegFilt5, srcRegFilt7));
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1,
_mm256_castsi256_si128(addFilterReg64));
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3,
_mm256_castsi256_si128(addFilterReg64));
// shift by 7 bit each 16 bit
srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7);
srcRegFilt3 = _mm_srai_epi16(srcRegFilt3, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt3);
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr, srcRegFilt1);
}
}
/*!
* \brief Fill a packed vector by replicating a value
*/
ETL_STATIC_INLINE(avx_simd_short) set(int16_t value) {
return _mm256_set1_epi16(value);
}
static void mb_lpf_horizontal_edge_w_avx2_16(unsigned char *s, int p,
const unsigned char *_blimit,
const unsigned char *_limit,
const unsigned char *_thresh) {
__m128i mask, hev, flat, flat2;
const __m128i zero = _mm_set1_epi16(0);
const __m128i one = _mm_set1_epi8(1);
__m128i p7, p6, p5;
__m128i p4, p3, p2, p1, p0, q0, q1, q2, q3, q4;
__m128i q5, q6, q7;
__m256i p256_7, q256_7, p256_6, q256_6, p256_5, q256_5, p256_4, q256_4,
p256_3, q256_3, p256_2, q256_2, p256_1, q256_1, p256_0, q256_0;
const __m128i thresh =
_mm_broadcastb_epi8(_mm_cvtsi32_si128((int)_thresh[0]));
const __m128i limit = _mm_broadcastb_epi8(_mm_cvtsi32_si128((int)_limit[0]));
const __m128i blimit =
_mm_broadcastb_epi8(_mm_cvtsi32_si128((int)_blimit[0]));
p256_4 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 5 * p)));
p256_3 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 4 * p)));
p256_2 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 3 * p)));
p256_1 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 2 * p)));
p256_0 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 1 * p)));
q256_0 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s - 0 * p)));
q256_1 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s + 1 * p)));
q256_2 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s + 2 * p)));
q256_3 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s + 3 * p)));
q256_4 =
_mm256_castpd_si256(_mm256_broadcast_pd((__m128d const *)(s + 4 * p)));
p4 = _mm256_castsi256_si128(p256_4);
p3 = _mm256_castsi256_si128(p256_3);
p2 = _mm256_castsi256_si128(p256_2);
p1 = _mm256_castsi256_si128(p256_1);
p0 = _mm256_castsi256_si128(p256_0);
q0 = _mm256_castsi256_si128(q256_0);
q1 = _mm256_castsi256_si128(q256_1);
q2 = _mm256_castsi256_si128(q256_2);
q3 = _mm256_castsi256_si128(q256_3);
q4 = _mm256_castsi256_si128(q256_4);
{
const __m128i abs_p1p0 =
_mm_or_si128(_mm_subs_epu8(p1, p0), _mm_subs_epu8(p0, p1));
const __m128i abs_q1q0 =
_mm_or_si128(_mm_subs_epu8(q1, q0), _mm_subs_epu8(q0, q1));
const __m128i fe = _mm_set1_epi8(0xfe);
const __m128i ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0);
__m128i abs_p0q0 =
_mm_or_si128(_mm_subs_epu8(p0, q0), _mm_subs_epu8(q0, p0));
__m128i abs_p1q1 =
_mm_or_si128(_mm_subs_epu8(p1, q1), _mm_subs_epu8(q1, p1));
__m128i work;
flat = _mm_max_epu8(abs_p1p0, abs_q1q0);
hev = _mm_subs_epu8(flat, thresh);
hev = _mm_xor_si128(_mm_cmpeq_epi8(hev, zero), ff);
abs_p0q0 = _mm_adds_epu8(abs_p0q0, abs_p0q0);
abs_p1q1 = _mm_srli_epi16(_mm_and_si128(abs_p1q1, fe), 1);
mask = _mm_subs_epu8(_mm_adds_epu8(abs_p0q0, abs_p1q1), blimit);
mask = _mm_xor_si128(_mm_cmpeq_epi8(mask, zero), ff);
// mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1;
mask = _mm_max_epu8(flat, mask);
// mask |= (abs(p1 - p0) > limit) * -1;
// mask |= (abs(q1 - q0) > limit) * -1;
work = _mm_max_epu8(
_mm_or_si128(_mm_subs_epu8(p2, p1), _mm_subs_epu8(p1, p2)),
_mm_or_si128(_mm_subs_epu8(p3, p2), _mm_subs_epu8(p2, p3)));
mask = _mm_max_epu8(work, mask);
work = _mm_max_epu8(
_mm_or_si128(_mm_subs_epu8(q2, q1), _mm_subs_epu8(q1, q2)),
_mm_or_si128(_mm_subs_epu8(q3, q2), _mm_subs_epu8(q2, q3)));
mask = _mm_max_epu8(work, mask);
mask = _mm_subs_epu8(mask, limit);
mask = _mm_cmpeq_epi8(mask, zero);
}
// lp filter
{
const __m128i t4 = _mm_set1_epi8(4);
const __m128i t3 = _mm_set1_epi8(3);
const __m128i t80 = _mm_set1_epi8(0x80);
const __m128i te0 = _mm_set1_epi8(0xe0);
const __m128i t1f = _mm_set1_epi8(0x1f);
const __m128i t1 = _mm_set1_epi8(0x1);
const __m128i t7f = _mm_set1_epi8(0x7f);
__m128i ps1 = _mm_xor_si128(p1, t80);
__m128i ps0 = _mm_xor_si128(p0, t80);
__m128i qs0 = _mm_xor_si128(q0, t80);
__m128i qs1 = _mm_xor_si128(q1, t80);
__m128i filt;
__m128i work_a;
__m128i filter1, filter2;
__m128i flat2_p6, flat2_p5, flat2_p4, flat2_p3, flat2_p2, flat2_p1,
flat2_p0, flat2_q0, flat2_q1, flat2_q2, flat2_q3, flat2_q4, flat2_q5,
flat2_q6, flat_p2, flat_p1, flat_p0, flat_q0, flat_q1, flat_q2;
filt = _mm_and_si128(_mm_subs_epi8(ps1, qs1), hev);
work_a = _mm_subs_epi8(qs0, ps0);
filt = _mm_adds_epi8(filt, work_a);
filt = _mm_adds_epi8(filt, work_a);
filt = _mm_adds_epi8(filt, work_a);
/* (vpx_filter + 3 * (qs0 - ps0)) & mask */
filt = _mm_and_si128(filt, mask);
filter1 = _mm_adds_epi8(filt, t4);
filter2 = _mm_adds_epi8(filt, t3);
/* Filter1 >> 3 */
work_a = _mm_cmpgt_epi8(zero, filter1);
filter1 = _mm_srli_epi16(filter1, 3);
work_a = _mm_and_si128(work_a, te0);
filter1 = _mm_and_si128(filter1, t1f);
filter1 = _mm_or_si128(filter1, work_a);
qs0 = _mm_xor_si128(_mm_subs_epi8(qs0, filter1), t80);
/* Filter2 >> 3 */
work_a = _mm_cmpgt_epi8(zero, filter2);
filter2 = _mm_srli_epi16(filter2, 3);
work_a = _mm_and_si128(work_a, te0);
filter2 = _mm_and_si128(filter2, t1f);
filter2 = _mm_or_si128(filter2, work_a);
ps0 = _mm_xor_si128(_mm_adds_epi8(ps0, filter2), t80);
/* filt >> 1 */
filt = _mm_adds_epi8(filter1, t1);
work_a = _mm_cmpgt_epi8(zero, filt);
filt = _mm_srli_epi16(filt, 1);
work_a = _mm_and_si128(work_a, t80);
filt = _mm_and_si128(filt, t7f);
filt = _mm_or_si128(filt, work_a);
filt = _mm_andnot_si128(hev, filt);
ps1 = _mm_xor_si128(_mm_adds_epi8(ps1, filt), t80);
qs1 = _mm_xor_si128(_mm_subs_epi8(qs1, filt), t80);
// loopfilter done
{
__m128i work;
work = _mm_max_epu8(
_mm_or_si128(_mm_subs_epu8(p2, p0), _mm_subs_epu8(p0, p2)),
_mm_or_si128(_mm_subs_epu8(q2, q0), _mm_subs_epu8(q0, q2)));
flat = _mm_max_epu8(work, flat);
work = _mm_max_epu8(
_mm_or_si128(_mm_subs_epu8(p3, p0), _mm_subs_epu8(p0, p3)),
_mm_or_si128(_mm_subs_epu8(q3, q0), _mm_subs_epu8(q0, q3)));
flat = _mm_max_epu8(work, flat);
work = _mm_max_epu8(
_mm_or_si128(_mm_subs_epu8(p4, p0), _mm_subs_epu8(p0, p4)),
_mm_or_si128(_mm_subs_epu8(q4, q0), _mm_subs_epu8(q0, q4)));
flat = _mm_subs_epu8(flat, one);
flat = _mm_cmpeq_epi8(flat, zero);
flat = _mm_and_si128(flat, mask);
p256_5 = _mm256_castpd_si256(
_mm256_broadcast_pd((__m128d const *)(s - 6 * p)));
q256_5 = _mm256_castpd_si256(
_mm256_broadcast_pd((__m128d const *)(s + 5 * p)));
p5 = _mm256_castsi256_si128(p256_5);
q5 = _mm256_castsi256_si128(q256_5);
flat2 = _mm_max_epu8(
_mm_or_si128(_mm_subs_epu8(p5, p0), _mm_subs_epu8(p0, p5)),
_mm_or_si128(_mm_subs_epu8(q5, q0), _mm_subs_epu8(q0, q5)));
flat2 = _mm_max_epu8(work, flat2);
p256_6 = _mm256_castpd_si256(
_mm256_broadcast_pd((__m128d const *)(s - 7 * p)));
q256_6 = _mm256_castpd_si256(
_mm256_broadcast_pd((__m128d const *)(s + 6 * p)));
p6 = _mm256_castsi256_si128(p256_6);
q6 = _mm256_castsi256_si128(q256_6);
work = _mm_max_epu8(
_mm_or_si128(_mm_subs_epu8(p6, p0), _mm_subs_epu8(p0, p6)),
_mm_or_si128(_mm_subs_epu8(q6, q0), _mm_subs_epu8(q0, q6)));
flat2 = _mm_max_epu8(work, flat2);
p256_7 = _mm256_castpd_si256(
_mm256_broadcast_pd((__m128d const *)(s - 8 * p)));
q256_7 = _mm256_castpd_si256(
_mm256_broadcast_pd((__m128d const *)(s + 7 * p)));
p7 = _mm256_castsi256_si128(p256_7);
q7 = _mm256_castsi256_si128(q256_7);
work = _mm_max_epu8(
_mm_or_si128(_mm_subs_epu8(p7, p0), _mm_subs_epu8(p0, p7)),
_mm_or_si128(_mm_subs_epu8(q7, q0), _mm_subs_epu8(q0, q7)));
flat2 = _mm_max_epu8(work, flat2);
flat2 = _mm_subs_epu8(flat2, one);
flat2 = _mm_cmpeq_epi8(flat2, zero);
flat2 = _mm_and_si128(flat2, flat); // flat2 & flat & mask
}
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// flat and wide flat calculations
{
const __m256i eight = _mm256_set1_epi16(8);
const __m256i four = _mm256_set1_epi16(4);
__m256i pixelFilter_p, pixelFilter_q, pixetFilter_p2p1p0,
pixetFilter_q2q1q0, sum_p7, sum_q7, sum_p3, sum_q3, res_p, res_q;
const __m256i filter =
_mm256_load_si256((__m256i const *)filt_loopfilter_avx2);
p256_7 = _mm256_shuffle_epi8(p256_7, filter);
p256_6 = _mm256_shuffle_epi8(p256_6, filter);
p256_5 = _mm256_shuffle_epi8(p256_5, filter);
p256_4 = _mm256_shuffle_epi8(p256_4, filter);
p256_3 = _mm256_shuffle_epi8(p256_3, filter);
p256_2 = _mm256_shuffle_epi8(p256_2, filter);
p256_1 = _mm256_shuffle_epi8(p256_1, filter);
p256_0 = _mm256_shuffle_epi8(p256_0, filter);
q256_0 = _mm256_shuffle_epi8(q256_0, filter);
q256_1 = _mm256_shuffle_epi8(q256_1, filter);
q256_2 = _mm256_shuffle_epi8(q256_2, filter);
q256_3 = _mm256_shuffle_epi8(q256_3, filter);
q256_4 = _mm256_shuffle_epi8(q256_4, filter);
q256_5 = _mm256_shuffle_epi8(q256_5, filter);
q256_6 = _mm256_shuffle_epi8(q256_6, filter);
q256_7 = _mm256_shuffle_epi8(q256_7, filter);
pixelFilter_p = _mm256_add_epi16(_mm256_add_epi16(p256_6, p256_5),
_mm256_add_epi16(p256_4, p256_3));
pixelFilter_q = _mm256_add_epi16(_mm256_add_epi16(q256_6, q256_5),
_mm256_add_epi16(q256_4, q256_3));
pixetFilter_p2p1p0 =
_mm256_add_epi16(p256_0, _mm256_add_epi16(p256_2, p256_1));
pixelFilter_p = _mm256_add_epi16(pixelFilter_p, pixetFilter_p2p1p0);
pixetFilter_q2q1q0 =
_mm256_add_epi16(q256_0, _mm256_add_epi16(q256_2, q256_1));
pixelFilter_q = _mm256_add_epi16(pixelFilter_q, pixetFilter_q2q1q0);
pixelFilter_p = _mm256_add_epi16(
eight, _mm256_add_epi16(pixelFilter_p, pixelFilter_q));
pixetFilter_p2p1p0 = _mm256_add_epi16(
four, _mm256_add_epi16(pixetFilter_p2p1p0, pixetFilter_q2q1q0));
res_p = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(p256_7, p256_0)), 4);
flat2_p0 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(q256_7, q256_0)), 4);
flat2_q0 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
res_p =
_mm256_srli_epi16(_mm256_add_epi16(pixetFilter_p2p1p0,
_mm256_add_epi16(p256_3, p256_0)),
3);
flat_p0 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q =
_mm256_srli_epi16(_mm256_add_epi16(pixetFilter_p2p1p0,
_mm256_add_epi16(q256_3, q256_0)),
3);
flat_q0 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
sum_p7 = _mm256_add_epi16(p256_7, p256_7);
sum_q7 = _mm256_add_epi16(q256_7, q256_7);
sum_p3 = _mm256_add_epi16(p256_3, p256_3);
sum_q3 = _mm256_add_epi16(q256_3, q256_3);
pixelFilter_q = _mm256_sub_epi16(pixelFilter_p, p256_6);
pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_6);
res_p = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_1)), 4);
flat2_p1 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_1)), 4);
flat2_q1 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
pixetFilter_q2q1q0 = _mm256_sub_epi16(pixetFilter_p2p1p0, p256_2);
pixetFilter_p2p1p0 = _mm256_sub_epi16(pixetFilter_p2p1p0, q256_2);
res_p =
_mm256_srli_epi16(_mm256_add_epi16(pixetFilter_p2p1p0,
_mm256_add_epi16(sum_p3, p256_1)),
3);
flat_p1 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q =
_mm256_srli_epi16(_mm256_add_epi16(pixetFilter_q2q1q0,
_mm256_add_epi16(sum_q3, q256_1)),
3);
flat_q1 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
sum_p7 = _mm256_add_epi16(sum_p7, p256_7);
sum_q7 = _mm256_add_epi16(sum_q7, q256_7);
sum_p3 = _mm256_add_epi16(sum_p3, p256_3);
sum_q3 = _mm256_add_epi16(sum_q3, q256_3);
pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_5);
pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_5);
res_p = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_2)), 4);
flat2_p2 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_2)), 4);
flat2_q2 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
pixetFilter_p2p1p0 = _mm256_sub_epi16(pixetFilter_p2p1p0, q256_1);
pixetFilter_q2q1q0 = _mm256_sub_epi16(pixetFilter_q2q1q0, p256_1);
res_p =
_mm256_srli_epi16(_mm256_add_epi16(pixetFilter_p2p1p0,
_mm256_add_epi16(sum_p3, p256_2)),
3);
flat_p2 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q =
_mm256_srli_epi16(_mm256_add_epi16(pixetFilter_q2q1q0,
_mm256_add_epi16(sum_q3, q256_2)),
3);
flat_q2 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
sum_p7 = _mm256_add_epi16(sum_p7, p256_7);
sum_q7 = _mm256_add_epi16(sum_q7, q256_7);
pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_4);
pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_4);
res_p = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_3)), 4);
flat2_p3 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_3)), 4);
flat2_q3 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
sum_p7 = _mm256_add_epi16(sum_p7, p256_7);
sum_q7 = _mm256_add_epi16(sum_q7, q256_7);
pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_3);
pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_3);
res_p = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_4)), 4);
flat2_p4 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_4)), 4);
flat2_q4 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
sum_p7 = _mm256_add_epi16(sum_p7, p256_7);
sum_q7 = _mm256_add_epi16(sum_q7, q256_7);
pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_2);
pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_2);
res_p = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_5)), 4);
flat2_p5 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_5)), 4);
flat2_q5 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
sum_p7 = _mm256_add_epi16(sum_p7, p256_7);
sum_q7 = _mm256_add_epi16(sum_q7, q256_7);
pixelFilter_p = _mm256_sub_epi16(pixelFilter_p, q256_1);
pixelFilter_q = _mm256_sub_epi16(pixelFilter_q, p256_1);
res_p = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_p, _mm256_add_epi16(sum_p7, p256_6)), 4);
flat2_p6 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_p, res_p), 168));
res_q = _mm256_srli_epi16(
_mm256_add_epi16(pixelFilter_q, _mm256_add_epi16(sum_q7, q256_6)), 4);
flat2_q6 = _mm256_castsi256_si128(
_mm256_permute4x64_epi64(_mm256_packus_epi16(res_q, res_q), 168));
}
// wide flat
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
p2 = _mm_andnot_si128(flat, p2);
flat_p2 = _mm_and_si128(flat, flat_p2);
p2 = _mm_or_si128(flat_p2, p2);
p1 = _mm_andnot_si128(flat, ps1);
flat_p1 = _mm_and_si128(flat, flat_p1);
p1 = _mm_or_si128(flat_p1, p1);
p0 = _mm_andnot_si128(flat, ps0);
flat_p0 = _mm_and_si128(flat, flat_p0);
p0 = _mm_or_si128(flat_p0, p0);
q0 = _mm_andnot_si128(flat, qs0);
flat_q0 = _mm_and_si128(flat, flat_q0);
q0 = _mm_or_si128(flat_q0, q0);
q1 = _mm_andnot_si128(flat, qs1);
flat_q1 = _mm_and_si128(flat, flat_q1);
q1 = _mm_or_si128(flat_q1, q1);
q2 = _mm_andnot_si128(flat, q2);
flat_q2 = _mm_and_si128(flat, flat_q2);
q2 = _mm_or_si128(flat_q2, q2);
p6 = _mm_andnot_si128(flat2, p6);
flat2_p6 = _mm_and_si128(flat2, flat2_p6);
p6 = _mm_or_si128(flat2_p6, p6);
_mm_storeu_si128((__m128i *)(s - 7 * p), p6);
p5 = _mm_andnot_si128(flat2, p5);
flat2_p5 = _mm_and_si128(flat2, flat2_p5);
p5 = _mm_or_si128(flat2_p5, p5);
_mm_storeu_si128((__m128i *)(s - 6 * p), p5);
p4 = _mm_andnot_si128(flat2, p4);
flat2_p4 = _mm_and_si128(flat2, flat2_p4);
p4 = _mm_or_si128(flat2_p4, p4);
_mm_storeu_si128((__m128i *)(s - 5 * p), p4);
p3 = _mm_andnot_si128(flat2, p3);
flat2_p3 = _mm_and_si128(flat2, flat2_p3);
p3 = _mm_or_si128(flat2_p3, p3);
_mm_storeu_si128((__m128i *)(s - 4 * p), p3);
p2 = _mm_andnot_si128(flat2, p2);
flat2_p2 = _mm_and_si128(flat2, flat2_p2);
p2 = _mm_or_si128(flat2_p2, p2);
_mm_storeu_si128((__m128i *)(s - 3 * p), p2);
p1 = _mm_andnot_si128(flat2, p1);
flat2_p1 = _mm_and_si128(flat2, flat2_p1);
p1 = _mm_or_si128(flat2_p1, p1);
_mm_storeu_si128((__m128i *)(s - 2 * p), p1);
p0 = _mm_andnot_si128(flat2, p0);
flat2_p0 = _mm_and_si128(flat2, flat2_p0);
p0 = _mm_or_si128(flat2_p0, p0);
_mm_storeu_si128((__m128i *)(s - 1 * p), p0);
q0 = _mm_andnot_si128(flat2, q0);
flat2_q0 = _mm_and_si128(flat2, flat2_q0);
q0 = _mm_or_si128(flat2_q0, q0);
_mm_storeu_si128((__m128i *)(s - 0 * p), q0);
q1 = _mm_andnot_si128(flat2, q1);
flat2_q1 = _mm_and_si128(flat2, flat2_q1);
q1 = _mm_or_si128(flat2_q1, q1);
_mm_storeu_si128((__m128i *)(s + 1 * p), q1);
q2 = _mm_andnot_si128(flat2, q2);
flat2_q2 = _mm_and_si128(flat2, flat2_q2);
q2 = _mm_or_si128(flat2_q2, q2);
_mm_storeu_si128((__m128i *)(s + 2 * p), q2);
q3 = _mm_andnot_si128(flat2, q3);
flat2_q3 = _mm_and_si128(flat2, flat2_q3);
q3 = _mm_or_si128(flat2_q3, q3);
_mm_storeu_si128((__m128i *)(s + 3 * p), q3);
q4 = _mm_andnot_si128(flat2, q4);
flat2_q4 = _mm_and_si128(flat2, flat2_q4);
q4 = _mm_or_si128(flat2_q4, q4);
_mm_storeu_si128((__m128i *)(s + 4 * p), q4);
q5 = _mm_andnot_si128(flat2, q5);
flat2_q5 = _mm_and_si128(flat2, flat2_q5);
q5 = _mm_or_si128(flat2_q5, q5);
_mm_storeu_si128((__m128i *)(s + 5 * p), q5);
q6 = _mm_andnot_si128(flat2, q6);
flat2_q6 = _mm_and_si128(flat2, flat2_q6);
q6 = _mm_or_si128(flat2_q6, q6);
_mm_storeu_si128((__m128i *)(s + 6 * p), q6);
}
}
int16_t * const restrict del_pr = _del_pr+PAD;
#ifdef PARASAIL_TABLE
parasail_result_t *result = parasail_result_new_table1(s1Len, s2Len);
#else
#ifdef PARASAIL_ROWCOL
parasail_result_t *result = parasail_result_new_rowcol1(s1Len, s2Len);
#else
parasail_result_t *result = parasail_result_new();
#endif
#endif
int32_t i = 0;
int32_t j = 0;
int16_t end_query = 0;
int16_t end_ref = 0;
int16_t score = NEG_INF;
__m256i vNegInf = _mm256_set1_epi16(NEG_INF);
__m256i vNegInf0 = _mm256_srli_si256_rpl(vNegInf, 2); /* shift in a 0 */
__m256i vOpen = _mm256_set1_epi16(open);
__m256i vGap = _mm256_set1_epi16(gap);
__m256i vZero = _mm256_set1_epi16(0);
__m256i vOne = _mm256_set1_epi16(1);
__m256i vN = _mm256_set1_epi16(N);
__m256i vNegOne = _mm256_set1_epi16(-1);
__m256i vI = _mm256_set_epi16(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15);
__m256i vJreset = _mm256_set_epi16(0,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15);
__m256i vMax = vNegInf;
__m256i vEndI = vNegInf;
__m256i vEndJ = vNegInf;
__m256i vILimit = _mm256_set1_epi16(s1Len);
__m256i vJLimit = _mm256_set1_epi16(s2Len);
const int32_t s1Len_PAD = s1Len+PAD; const int32_t s2Len_PAD = s2Len+PAD; int16_t * const restrict s1 = parasail_memalign_int16_t(32, s1Len+PAD); int16_t * const restrict s2B= parasail_memalign_int16_t(32, s2Len+PAD2); int16_t * const restrict _H_pr = parasail_memalign_int16_t(32, s2Len+PAD2); int16_t * const restrict _F_pr = parasail_memalign_int16_t(32, s2Len+PAD2); int16_t * const restrict s2 = s2B+PAD; /* will allow later for negative indices */ int16_t * const restrict H_pr = _H_pr+PAD; int16_t * const restrict F_pr = _F_pr+PAD; parasail_result_t *result = parasail_result_new_trace(s1Len, s2Len, 32, sizeof(int8_t)); int32_t i = 0; int32_t j = 0; int32_t end_query = 0; int32_t end_ref = 0; int16_t score = NEG_INF; __m256i vNegInf = _mm256_set1_epi16(NEG_INF); __m256i vNegInf0 = _mm256_srli_si256_rpl(vNegInf, 2); /* shift in a 0 */ __m256i vOpen = _mm256_set1_epi16(open); __m256i vGap = _mm256_set1_epi16(gap); __m256i vZero = _mm256_set1_epi16(0); __m256i vOne = _mm256_set1_epi16(1); __m256i vN = _mm256_set1_epi16(N); __m256i vNegOne = _mm256_set1_epi16(-1); __m256i vI = _mm256_set_epi16(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15); __m256i vJreset = _mm256_set_epi16(0,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15); __m256i vMaxH = vNegInf; __m256i vEndI = vNegInf; __m256i vEndJ = vNegInf; __m256i vILimit = _mm256_set1_epi16(s1Len); __m256i vJLimit = _mm256_set1_epi16(s2Len); __m256i vTDiag = _mm256_set1_epi16(PARASAIL_DIAG);
INLINE static void haddwd_accumulate_dual_avx2(__m256i *accumulate, __m256i *ver_row)
{
__m256i abs_value = _mm256_abs_epi16(*ver_row);
*accumulate = _mm256_add_epi32(*accumulate, _mm256_madd_epi16(abs_value, _mm256_set1_epi16(1)));
}
static INLINE void build_compound_diffwtd_mask_d16_inv_avx2(
uint8_t *mask, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, int h, int w, int shift) {
const int mask_base = 38;
const __m256i _r = _mm256_set1_epi16((1 << shift) >> 1);
const __m256i y38 = _mm256_set1_epi16(mask_base);
const __m256i y64 = _mm256_set1_epi16(AOM_BLEND_A64_MAX_ALPHA);
int i = 0;
if (w == 4) {
do {
const __m128i s0A = xx_loadl_64(src0);
const __m128i s0B = xx_loadl_64(src0 + src0_stride);
const __m128i s0C = xx_loadl_64(src0 + src0_stride * 2);
const __m128i s0D = xx_loadl_64(src0 + src0_stride * 3);
const __m128i s1A = xx_loadl_64(src1);
const __m128i s1B = xx_loadl_64(src1 + src1_stride);
const __m128i s1C = xx_loadl_64(src1 + src1_stride * 2);
const __m128i s1D = xx_loadl_64(src1 + src1_stride * 3);
const __m256i s0 = yy_set_m128i(_mm_unpacklo_epi64(s0C, s0D),
_mm_unpacklo_epi64(s0A, s0B));
const __m256i s1 = yy_set_m128i(_mm_unpacklo_epi64(s1C, s1D),
_mm_unpacklo_epi64(s1A, s1B));
const __m256i m16 =
calc_mask_d16_inv_avx2(&s0, &s1, &_r, &y38, &y64, shift);
const __m256i m8 = _mm256_packus_epi16(m16, _mm256_setzero_si256());
xx_storeu_128(mask,
_mm256_castsi256_si128(_mm256_permute4x64_epi64(m8, 0xd8)));
src0 += src0_stride << 2;
src1 += src1_stride << 2;
mask += 16;
i += 4;
} while (i < h);
} else if (w == 8) {
do {
const __m256i s0AB = yy_loadu2_128(src0 + src0_stride, src0);
const __m256i s0CD =
yy_loadu2_128(src0 + src0_stride * 3, src0 + src0_stride * 2);
const __m256i s1AB = yy_loadu2_128(src1 + src1_stride, src1);
const __m256i s1CD =
yy_loadu2_128(src1 + src1_stride * 3, src1 + src1_stride * 2);
const __m256i m16AB =
calc_mask_d16_inv_avx2(&s0AB, &s1AB, &_r, &y38, &y64, shift);
const __m256i m16CD =
calc_mask_d16_inv_avx2(&s0CD, &s1CD, &_r, &y38, &y64, shift);
const __m256i m8 = _mm256_packus_epi16(m16AB, m16CD);
yy_storeu_256(mask, _mm256_permute4x64_epi64(m8, 0xd8));
src0 += src0_stride << 2;
src1 += src1_stride << 2;
mask += 32;
i += 4;
} while (i < h);
} else if (w == 16) {
do {
const __m256i s0A = yy_loadu_256(src0);
const __m256i s0B = yy_loadu_256(src0 + src0_stride);
const __m256i s1A = yy_loadu_256(src1);
const __m256i s1B = yy_loadu_256(src1 + src1_stride);
const __m256i m16A =
calc_mask_d16_inv_avx2(&s0A, &s1A, &_r, &y38, &y64, shift);
const __m256i m16B =
calc_mask_d16_inv_avx2(&s0B, &s1B, &_r, &y38, &y64, shift);
const __m256i m8 = _mm256_packus_epi16(m16A, m16B);
yy_storeu_256(mask, _mm256_permute4x64_epi64(m8, 0xd8));
src0 += src0_stride << 1;
src1 += src1_stride << 1;
mask += 32;
i += 2;
} while (i < h);
} else if (w == 32) {
do {
const __m256i s0A = yy_loadu_256(src0);
const __m256i s0B = yy_loadu_256(src0 + 16);
const __m256i s1A = yy_loadu_256(src1);
const __m256i s1B = yy_loadu_256(src1 + 16);
const __m256i m16A =
calc_mask_d16_inv_avx2(&s0A, &s1A, &_r, &y38, &y64, shift);
const __m256i m16B =
calc_mask_d16_inv_avx2(&s0B, &s1B, &_r, &y38, &y64, shift);
const __m256i m8 = _mm256_packus_epi16(m16A, m16B);
yy_storeu_256(mask, _mm256_permute4x64_epi64(m8, 0xd8));
src0 += src0_stride;
src1 += src1_stride;
mask += 32;
i += 1;
} while (i < h);
} else if (w == 64) {
do {
const __m256i s0A = yy_loadu_256(src0);
const __m256i s0B = yy_loadu_256(src0 + 16);
const __m256i s0C = yy_loadu_256(src0 + 32);
const __m256i s0D = yy_loadu_256(src0 + 48);
const __m256i s1A = yy_loadu_256(src1);
const __m256i s1B = yy_loadu_256(src1 + 16);
const __m256i s1C = yy_loadu_256(src1 + 32);
const __m256i s1D = yy_loadu_256(src1 + 48);
const __m256i m16A =
calc_mask_d16_inv_avx2(&s0A, &s1A, &_r, &y38, &y64, shift);
const __m256i m16B =
calc_mask_d16_inv_avx2(&s0B, &s1B, &_r, &y38, &y64, shift);
const __m256i m16C =
calc_mask_d16_inv_avx2(&s0C, &s1C, &_r, &y38, &y64, shift);
const __m256i m16D =
calc_mask_d16_inv_avx2(&s0D, &s1D, &_r, &y38, &y64, shift);
const __m256i m8AB = _mm256_packus_epi16(m16A, m16B);
const __m256i m8CD = _mm256_packus_epi16(m16C, m16D);
yy_storeu_256(mask, _mm256_permute4x64_epi64(m8AB, 0xd8));
yy_storeu_256(mask + 32, _mm256_permute4x64_epi64(m8CD, 0xd8));
src0 += src0_stride;
src1 += src1_stride;
mask += 64;
i += 1;
} while (i < h);
} else {
do {
const __m256i s0A = yy_loadu_256(src0);
const __m256i s0B = yy_loadu_256(src0 + 16);
const __m256i s0C = yy_loadu_256(src0 + 32);
const __m256i s0D = yy_loadu_256(src0 + 48);
const __m256i s0E = yy_loadu_256(src0 + 64);
const __m256i s0F = yy_loadu_256(src0 + 80);
const __m256i s0G = yy_loadu_256(src0 + 96);
const __m256i s0H = yy_loadu_256(src0 + 112);
const __m256i s1A = yy_loadu_256(src1);
const __m256i s1B = yy_loadu_256(src1 + 16);
const __m256i s1C = yy_loadu_256(src1 + 32);
const __m256i s1D = yy_loadu_256(src1 + 48);
const __m256i s1E = yy_loadu_256(src1 + 64);
const __m256i s1F = yy_loadu_256(src1 + 80);
const __m256i s1G = yy_loadu_256(src1 + 96);
const __m256i s1H = yy_loadu_256(src1 + 112);
const __m256i m16A =
calc_mask_d16_inv_avx2(&s0A, &s1A, &_r, &y38, &y64, shift);
const __m256i m16B =
calc_mask_d16_inv_avx2(&s0B, &s1B, &_r, &y38, &y64, shift);
const __m256i m16C =
calc_mask_d16_inv_avx2(&s0C, &s1C, &_r, &y38, &y64, shift);
const __m256i m16D =
calc_mask_d16_inv_avx2(&s0D, &s1D, &_r, &y38, &y64, shift);
const __m256i m16E =
calc_mask_d16_inv_avx2(&s0E, &s1E, &_r, &y38, &y64, shift);
const __m256i m16F =
calc_mask_d16_inv_avx2(&s0F, &s1F, &_r, &y38, &y64, shift);
const __m256i m16G =
calc_mask_d16_inv_avx2(&s0G, &s1G, &_r, &y38, &y64, shift);
const __m256i m16H =
calc_mask_d16_inv_avx2(&s0H, &s1H, &_r, &y38, &y64, shift);
const __m256i m8AB = _mm256_packus_epi16(m16A, m16B);
const __m256i m8CD = _mm256_packus_epi16(m16C, m16D);
const __m256i m8EF = _mm256_packus_epi16(m16E, m16F);
const __m256i m8GH = _mm256_packus_epi16(m16G, m16H);
yy_storeu_256(mask, _mm256_permute4x64_epi64(m8AB, 0xd8));
yy_storeu_256(mask + 32, _mm256_permute4x64_epi64(m8CD, 0xd8));
yy_storeu_256(mask + 64, _mm256_permute4x64_epi64(m8EF, 0xd8));
yy_storeu_256(mask + 96, _mm256_permute4x64_epi64(m8GH, 0xd8));
src0 += src0_stride;
src1 += src1_stride;
mask += 128;
i += 1;
} while (i < h);
}
}
static void vpx_filter_block1d16_h8_avx2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr,
ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m256i addFilterReg64, filt1Reg, filt2Reg, filt3Reg, filt4Reg;
__m256i firstFilters, secondFilters, thirdFilters, forthFilters;
__m256i srcRegFilt32b1_1, srcRegFilt32b2_1, srcRegFilt32b2, srcRegFilt32b3;
__m256i srcReg32b1, srcReg32b2, filtersReg32;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
// create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
addFilterReg64 = _mm256_set1_epi32((int)0x0400040u);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg =_mm_packs_epi16(filtersReg, filtersReg);
// have the same data in both lanes of a 256 bit register
filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
// duplicate only the first 16 bits (first and second byte)
// across 256 bit register
firstFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x100u));
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x504u));
// duplicate only the forth 16 bits (seventh and eighth byte)
// across 256 bit register
forthFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x706u));
filt1Reg = _mm256_load_si256((__m256i const *)filt1_global_avx2);
filt2Reg = _mm256_load_si256((__m256i const *)filt2_global_avx2);
filt3Reg = _mm256_load_si256((__m256i const *)filt3_global_avx2);
filt4Reg = _mm256_load_si256((__m256i const *)filt4_global_avx2);
// multiple the size of the source and destination stride by two
src_stride = src_pixels_per_line << 1;
dst_stride = output_pitch << 1;
for (i = output_height; i > 1; i-=2) {
// load the 2 strides of source
srcReg32b1 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr - 3)));
srcReg32b1 = _mm256_inserti128_si256(srcReg32b1,
_mm_loadu_si128((const __m128i *)
(src_ptr+src_pixels_per_line-3)), 1);
// filter the source buffer
srcRegFilt32b1_1= _mm256_shuffle_epi8(srcReg32b1, filt1Reg);
srcRegFilt32b2= _mm256_shuffle_epi8(srcReg32b1, filt4Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b1_1 = _mm256_maddubs_epi16(srcRegFilt32b1_1, firstFilters);
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, forthFilters);
// add and saturate the results together
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, srcRegFilt32b2);
// filter the source buffer
srcRegFilt32b3= _mm256_shuffle_epi8(srcReg32b1, filt2Reg);
srcRegFilt32b2= _mm256_shuffle_epi8(srcReg32b1, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters);
// add and saturate the results together
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1,
_mm256_min_epi16(srcRegFilt32b3, srcRegFilt32b2));
// reading 2 strides of the next 16 bytes
// (part of it was being read by earlier read)
srcReg32b2 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + 5)));
srcReg32b2 = _mm256_inserti128_si256(srcReg32b2,
_mm_loadu_si128((const __m128i *)
(src_ptr+src_pixels_per_line+5)), 1);
// add and saturate the results together
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1,
_mm256_max_epi16(srcRegFilt32b3, srcRegFilt32b2));
// filter the source buffer
srcRegFilt32b2_1 = _mm256_shuffle_epi8(srcReg32b2, filt1Reg);
srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b2, filt4Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b2_1 = _mm256_maddubs_epi16(srcRegFilt32b2_1, firstFilters);
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, forthFilters);
// add and saturate the results together
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, srcRegFilt32b2);
// filter the source buffer
srcRegFilt32b3= _mm256_shuffle_epi8(srcReg32b2, filt2Reg);
srcRegFilt32b2= _mm256_shuffle_epi8(srcReg32b2, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters);
// add and saturate the results together
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1,
_mm256_min_epi16(srcRegFilt32b3, srcRegFilt32b2));
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1,
_mm256_max_epi16(srcRegFilt32b3, srcRegFilt32b2));
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, addFilterReg64);
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, addFilterReg64);
// shift by 7 bit each 16 bit
srcRegFilt32b1_1 = _mm256_srai_epi16(srcRegFilt32b1_1, 7);
srcRegFilt32b2_1 = _mm256_srai_epi16(srcRegFilt32b2_1, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcRegFilt32b1_1 = _mm256_packus_epi16(srcRegFilt32b1_1,
srcRegFilt32b2_1);
src_ptr+=src_stride;
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr,
_mm256_castsi256_si128(srcRegFilt32b1_1));
// save the next 16 bits
_mm_store_si128((__m128i*)(output_ptr+output_pitch),
_mm256_extractf128_si256(srcRegFilt32b1_1, 1));
output_ptr+=dst_stride;
}
// if the number of strides is odd.
// process only 16 bytes
if (i > 0) {
__m128i srcReg1, srcReg2, srcRegFilt1_1, srcRegFilt2_1;
__m128i srcRegFilt2, srcRegFilt3;
srcReg1 = _mm_loadu_si128((const __m128i *)(src_ptr - 3));
// filter the source buffer
srcRegFilt1_1 = _mm_shuffle_epi8(srcReg1,
_mm256_castsi256_si128(filt1Reg));
srcRegFilt2 = _mm_shuffle_epi8(srcReg1,
_mm256_castsi256_si128(filt4Reg));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt1_1 = _mm_maddubs_epi16(srcRegFilt1_1,
_mm256_castsi256_si128(firstFilters));
srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2,
_mm256_castsi256_si128(forthFilters));
// add and saturate the results together
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1, srcRegFilt2);
// filter the source buffer
srcRegFilt3= _mm_shuffle_epi8(srcReg1,
_mm256_castsi256_si128(filt2Reg));
srcRegFilt2= _mm_shuffle_epi8(srcReg1,
_mm256_castsi256_si128(filt3Reg));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt3 = _mm_maddubs_epi16(srcRegFilt3,
_mm256_castsi256_si128(secondFilters));
srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2,
_mm256_castsi256_si128(thirdFilters));
// add and saturate the results together
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1,
_mm_min_epi16(srcRegFilt3, srcRegFilt2));
// reading the next 16 bytes
// (part of it was being read by earlier read)
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + 5));
// add and saturate the results together
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1,
_mm_max_epi16(srcRegFilt3, srcRegFilt2));
// filter the source buffer
srcRegFilt2_1 = _mm_shuffle_epi8(srcReg2,
_mm256_castsi256_si128(filt1Reg));
srcRegFilt2 = _mm_shuffle_epi8(srcReg2,
_mm256_castsi256_si128(filt4Reg));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt2_1 = _mm_maddubs_epi16(srcRegFilt2_1,
_mm256_castsi256_si128(firstFilters));
srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2,
_mm256_castsi256_si128(forthFilters));
// add and saturate the results together
srcRegFilt2_1 = _mm_adds_epi16(srcRegFilt2_1, srcRegFilt2);
// filter the source buffer
srcRegFilt3 = _mm_shuffle_epi8(srcReg2,
_mm256_castsi256_si128(filt2Reg));
srcRegFilt2 = _mm_shuffle_epi8(srcReg2,
_mm256_castsi256_si128(filt3Reg));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt3 = _mm_maddubs_epi16(srcRegFilt3,
_mm256_castsi256_si128(secondFilters));
srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2,
_mm256_castsi256_si128(thirdFilters));
// add and saturate the results together
srcRegFilt2_1 = _mm_adds_epi16(srcRegFilt2_1,
_mm_min_epi16(srcRegFilt3, srcRegFilt2));
srcRegFilt2_1 = _mm_adds_epi16(srcRegFilt2_1,
_mm_max_epi16(srcRegFilt3, srcRegFilt2));
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1,
_mm256_castsi256_si128(addFilterReg64));
srcRegFilt2_1 = _mm_adds_epi16(srcRegFilt2_1,
_mm256_castsi256_si128(addFilterReg64));
// shift by 7 bit each 16 bit
srcRegFilt1_1 = _mm_srai_epi16(srcRegFilt1_1, 7);
srcRegFilt2_1 = _mm_srai_epi16(srcRegFilt2_1, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcRegFilt1_1 = _mm_packus_epi16(srcRegFilt1_1, srcRegFilt2_1);
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr, srcRegFilt1_1);
}
}
void aom_sad64x64x4d_avx2(const uint8_t *src, int src_stride,
const uint8_t *const ref[4], int ref_stride,
uint32_t res[4]) {
__m256i src_reg, srcnext_reg, ref0_reg, ref0next_reg;
__m256i ref1_reg, ref1next_reg, ref2_reg, ref2next_reg;
__m256i ref3_reg, ref3next_reg;
__m256i sum_ref0, sum_ref1, sum_ref2, sum_ref3;
__m256i sum_mlow, sum_mhigh;
int i;
const uint8_t *ref0, *ref1, *ref2, *ref3;
ref0 = ref[0];
ref1 = ref[1];
ref2 = ref[2];
ref3 = ref[3];
sum_ref0 = _mm256_set1_epi16(0);
sum_ref1 = _mm256_set1_epi16(0);
sum_ref2 = _mm256_set1_epi16(0);
sum_ref3 = _mm256_set1_epi16(0);
for (i = 0; i < 64; i++) {
// load 64 bytes from src and all refs
src_reg = _mm256_loadu_si256((const __m256i *)src);
srcnext_reg = _mm256_loadu_si256((const __m256i *)(src + 32));
ref0_reg = _mm256_loadu_si256((const __m256i *)ref0);
ref0next_reg = _mm256_loadu_si256((const __m256i *)(ref0 + 32));
ref1_reg = _mm256_loadu_si256((const __m256i *)ref1);
ref1next_reg = _mm256_loadu_si256((const __m256i *)(ref1 + 32));
ref2_reg = _mm256_loadu_si256((const __m256i *)ref2);
ref2next_reg = _mm256_loadu_si256((const __m256i *)(ref2 + 32));
ref3_reg = _mm256_loadu_si256((const __m256i *)ref3);
ref3next_reg = _mm256_loadu_si256((const __m256i *)(ref3 + 32));
// sum of the absolute differences between every ref-i to src
ref0_reg = _mm256_sad_epu8(ref0_reg, src_reg);
ref1_reg = _mm256_sad_epu8(ref1_reg, src_reg);
ref2_reg = _mm256_sad_epu8(ref2_reg, src_reg);
ref3_reg = _mm256_sad_epu8(ref3_reg, src_reg);
ref0next_reg = _mm256_sad_epu8(ref0next_reg, srcnext_reg);
ref1next_reg = _mm256_sad_epu8(ref1next_reg, srcnext_reg);
ref2next_reg = _mm256_sad_epu8(ref2next_reg, srcnext_reg);
ref3next_reg = _mm256_sad_epu8(ref3next_reg, srcnext_reg);
// sum every ref-i
sum_ref0 = _mm256_add_epi32(sum_ref0, ref0_reg);
sum_ref1 = _mm256_add_epi32(sum_ref1, ref1_reg);
sum_ref2 = _mm256_add_epi32(sum_ref2, ref2_reg);
sum_ref3 = _mm256_add_epi32(sum_ref3, ref3_reg);
sum_ref0 = _mm256_add_epi32(sum_ref0, ref0next_reg);
sum_ref1 = _mm256_add_epi32(sum_ref1, ref1next_reg);
sum_ref2 = _mm256_add_epi32(sum_ref2, ref2next_reg);
sum_ref3 = _mm256_add_epi32(sum_ref3, ref3next_reg);
src += src_stride;
ref0 += ref_stride;
ref1 += ref_stride;
ref2 += ref_stride;
ref3 += ref_stride;
}
{
__m128i sum;
// in sum_ref-i the result is saved in the first 4 bytes
// the other 4 bytes are zeroed.
// sum_ref1 and sum_ref3 are shifted left by 4 bytes
sum_ref1 = _mm256_slli_si256(sum_ref1, 4);
sum_ref3 = _mm256_slli_si256(sum_ref3, 4);
// merge sum_ref0 and sum_ref1 also sum_ref2 and sum_ref3
sum_ref0 = _mm256_or_si256(sum_ref0, sum_ref1);
sum_ref2 = _mm256_or_si256(sum_ref2, sum_ref3);
// merge every 64 bit from each sum_ref-i
sum_mlow = _mm256_unpacklo_epi64(sum_ref0, sum_ref2);
sum_mhigh = _mm256_unpackhi_epi64(sum_ref0, sum_ref2);
// add the low 64 bit to the high 64 bit
sum_mlow = _mm256_add_epi32(sum_mlow, sum_mhigh);
// add the low 128 bit to the high 128 bit
sum = _mm_add_epi32(_mm256_castsi256_si128(sum_mlow),
_mm256_extractf128_si256(sum_mlow, 1));
_mm_storeu_si128((__m128i *)(res), sum);
}
_mm256_zeroupper();
}
void av1_build_compound_diffwtd_mask_highbd_avx2(
uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const uint8_t *src0,
int src0_stride, const uint8_t *src1, int src1_stride, int h, int w,
int bd) {
if (w < 16) {
av1_build_compound_diffwtd_mask_highbd_ssse3(
mask, mask_type, src0, src0_stride, src1, src1_stride, h, w, bd);
} else {
assert(mask_type == DIFFWTD_38 || mask_type == DIFFWTD_38_INV);
assert(bd >= 8);
assert((w % 16) == 0);
const __m256i y0 = _mm256_setzero_si256();
const __m256i yAOM_BLEND_A64_MAX_ALPHA =
_mm256_set1_epi16(AOM_BLEND_A64_MAX_ALPHA);
const int mask_base = 38;
const __m256i ymask_base = _mm256_set1_epi16(mask_base);
const uint16_t *ssrc0 = CONVERT_TO_SHORTPTR(src0);
const uint16_t *ssrc1 = CONVERT_TO_SHORTPTR(src1);
if (bd == 8) {
if (mask_type == DIFFWTD_38_INV) {
for (int i = 0; i < h; ++i) {
for (int j = 0; j < w; j += 16) {
__m256i s0 = _mm256_loadu_si256((const __m256i *)&ssrc0[j]);
__m256i s1 = _mm256_loadu_si256((const __m256i *)&ssrc1[j]);
__m256i diff = _mm256_srai_epi16(
_mm256_abs_epi16(_mm256_sub_epi16(s0, s1)), DIFF_FACTOR_LOG2);
__m256i m = _mm256_min_epi16(
_mm256_max_epi16(y0, _mm256_add_epi16(diff, ymask_base)),
yAOM_BLEND_A64_MAX_ALPHA);
m = _mm256_sub_epi16(yAOM_BLEND_A64_MAX_ALPHA, m);
m = _mm256_packus_epi16(m, m);
m = _mm256_permute4x64_epi64(m, _MM_SHUFFLE(0, 0, 2, 0));
__m128i m0 = _mm256_castsi256_si128(m);
_mm_storeu_si128((__m128i *)&mask[j], m0);
}
ssrc0 += src0_stride;
ssrc1 += src1_stride;
mask += w;
}
} else {
for (int i = 0; i < h; ++i) {
for (int j = 0; j < w; j += 16) {
__m256i s0 = _mm256_loadu_si256((const __m256i *)&ssrc0[j]);
__m256i s1 = _mm256_loadu_si256((const __m256i *)&ssrc1[j]);
__m256i diff = _mm256_srai_epi16(
_mm256_abs_epi16(_mm256_sub_epi16(s0, s1)), DIFF_FACTOR_LOG2);
__m256i m = _mm256_min_epi16(
_mm256_max_epi16(y0, _mm256_add_epi16(diff, ymask_base)),
yAOM_BLEND_A64_MAX_ALPHA);
m = _mm256_packus_epi16(m, m);
m = _mm256_permute4x64_epi64(m, _MM_SHUFFLE(0, 0, 2, 0));
__m128i m0 = _mm256_castsi256_si128(m);
_mm_storeu_si128((__m128i *)&mask[j], m0);
}
ssrc0 += src0_stride;
ssrc1 += src1_stride;
mask += w;
}
}
} else {
const __m128i xshift = xx_set1_64_from_32i(bd - 8 + DIFF_FACTOR_LOG2);
if (mask_type == DIFFWTD_38_INV) {
for (int i = 0; i < h; ++i) {
for (int j = 0; j < w; j += 16) {
__m256i s0 = _mm256_loadu_si256((const __m256i *)&ssrc0[j]);
__m256i s1 = _mm256_loadu_si256((const __m256i *)&ssrc1[j]);
__m256i diff = _mm256_sra_epi16(
_mm256_abs_epi16(_mm256_sub_epi16(s0, s1)), xshift);
__m256i m = _mm256_min_epi16(
_mm256_max_epi16(y0, _mm256_add_epi16(diff, ymask_base)),
yAOM_BLEND_A64_MAX_ALPHA);
m = _mm256_sub_epi16(yAOM_BLEND_A64_MAX_ALPHA, m);
m = _mm256_packus_epi16(m, m);
m = _mm256_permute4x64_epi64(m, _MM_SHUFFLE(0, 0, 2, 0));
__m128i m0 = _mm256_castsi256_si128(m);
_mm_storeu_si128((__m128i *)&mask[j], m0);
}
ssrc0 += src0_stride;
ssrc1 += src1_stride;
mask += w;
}
} else {
for (int i = 0; i < h; ++i) {
for (int j = 0; j < w; j += 16) {
__m256i s0 = _mm256_loadu_si256((const __m256i *)&ssrc0[j]);
__m256i s1 = _mm256_loadu_si256((const __m256i *)&ssrc1[j]);
__m256i diff = _mm256_sra_epi16(
_mm256_abs_epi16(_mm256_sub_epi16(s0, s1)), xshift);
__m256i m = _mm256_min_epi16(
_mm256_max_epi16(y0, _mm256_add_epi16(diff, ymask_base)),
yAOM_BLEND_A64_MAX_ALPHA);
m = _mm256_packus_epi16(m, m);
m = _mm256_permute4x64_epi64(m, _MM_SHUFFLE(0, 0, 2, 0));
__m128i m0 = _mm256_castsi256_si128(m);
_mm_storeu_si128((__m128i *)&mask[j], m0);
}
ssrc0 += src0_stride;
ssrc1 += src1_stride;
mask += w;
}
}
}
}
}
void Scaler::process_plane_flt_avx2 (typename DST::Ptr::Type dst_ptr, typename SRC::PtrConst::Type src_ptr, int dst_stride, int src_stride, int width, int y_dst_beg, int y_dst_end) const
{
assert (DST::Ptr::check_ptr (dst_ptr, DST::ALIGN_W));
assert (SRC::PtrConst::check_ptr (src_ptr, SRC::ALIGN_R));
// When the destination is a buffer:
// mod4 is enough to guarantee alignment, but since we process pairs of
// vectors and write_partial() is not different from write() with float
// data (overwriting all the 64 bytes), we must take extra-care not to
// overflow from the output buffer.
// When the destination is not a buffer (output frame data), data may be
// unaligned anyway. (TO DO: check the algorithm and make sure this
// constraint is actual).
assert ((dst_stride & 15) == 0);
assert ((src_stride & 3) == 0);
assert (width > 0);
assert (y_dst_beg >= 0);
assert (y_dst_beg < y_dst_end);
assert (y_dst_end <= _dst_height);
const __m256i zero = _mm256_setzero_si256 ();
const __m256i mask_lsb = _mm256_set1_epi16 (0x00FF);
const __m256i sign_bit = _mm256_set1_epi16 (-0x8000);
const __m256 offset = _mm256_set1_ps (float (DST::OFFSET));
const __m256 add_cst = _mm256_set1_ps (float (_add_cst_flt));
const int w16 = width & -16;
const int w15 = width - w16;
for (int y = y_dst_beg; y < y_dst_end; ++y)
{
const KernelInfo& kernel_info = _kernel_info_arr [y];
const int kernel_size = kernel_info._kernel_size;
const float * coef_base_ptr = &_coef_flt_arr [kernel_info._coef_index];
const int ofs_y = kernel_info._start_line;
typename SRC::PtrConst::Type col_src_ptr = src_ptr;
SRC::PtrConst::jump (col_src_ptr, src_stride * ofs_y);
typename DST::Ptr::Type col_dst_ptr = dst_ptr;
typedef ScalerCopy <DST, 0, SRC, 0> ScCopy;
if (ScCopy::can_copy (kernel_info._copy_flt_flag))
{
ScCopy::copy (col_dst_ptr, col_src_ptr, width);
}
else
{
__m256 sum0;
__m256 sum1;
for (int x = 0; x < w16; x += 16)
{
typename SRC::PtrConst::Type pix_ptr = col_src_ptr;
process_vect_flt_avx2 <SRC, false> (
sum0, sum1, kernel_size, coef_base_ptr,
pix_ptr, zero, src_stride, add_cst, 0
);
DST::write_flt (
col_dst_ptr, sum0, sum1, mask_lsb, sign_bit, offset
);
DST::Ptr::jump (col_dst_ptr, 16);
SRC::PtrConst::jump (col_src_ptr, 16);
}
if (w15 > 0)
{
typename SRC::PtrConst::Type pix_ptr = col_src_ptr;
process_vect_flt_avx2 <SRC, true> (
sum0, sum1, kernel_size, coef_base_ptr,
pix_ptr, zero, src_stride, add_cst, w15
);
DST::write_flt_partial (
col_dst_ptr, sum0, sum1, mask_lsb, sign_bit, offset, w15
);
}
}
DST::Ptr::jump (dst_ptr, dst_stride);
}
_mm256_zeroupper (); // Back to SSE state
}
