int main()
{
//Transpose
vec4 mat[4] = {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}, {13, 14, 15, 16}};
__m128i xmm0 = _mm_unpacklo_epi32(_mm_castps_si128(mat[0]), _mm_castps_si128(mat[1]));
__m128i xmm1 = _mm_unpackhi_epi32(_mm_castps_si128(mat[0]), _mm_castps_si128(mat[1]));
__m128i xmm2 = _mm_unpacklo_epi32(_mm_castps_si128(mat[2]), _mm_castps_si128(mat[3]));
__m128i xmm3 = _mm_unpackhi_epi32(_mm_castps_si128(mat[2]), _mm_castps_si128(mat[3]));
vec4 trans[4];
trans[0] = _mm_castsi128_ps(_mm_unpacklo_epi64(xmm0, xmm2));
trans[1] = _mm_castsi128_ps(_mm_unpackhi_epi64(xmm0, xmm2));
trans[2] = _mm_castsi128_ps(_mm_unpacklo_epi64(xmm1, xmm3));
trans[3] = _mm_castsi128_ps(_mm_unpackhi_epi64(xmm1, xmm3));
vec4 trans2[4];
ml::transpose(trans2, mat);
FILE* file = fopen("..\\..\\AppData\\VT.swf", "rb");
fseek(file, 0, SEEK_END);
size_t size = ftell(file);
fseek(file, 0, SEEK_SET);
unsigned char* fileData = (unsigned char*)malloc(size);
fread(fileData, 1, size, file);
fclose(file);
MemReader data = {(const char*)fileData, (const char*)fileData+size, (const char*)fileData};
//Read SWF header
const u32 signatureAndVersion = data.read<u32>();
const u32 actualSize = data.read<u32>();
u32 signature = signatureAndVersion&0x00FFFFFF;
u8 version = signatureAndVersion>>24;
bool isCompressed = signature=='\0SWC';
bool isUncompressed = signature=='\0SWF';
//if !isCompressed && !isUncompressed return error;
MemReader data2 = {0, 0, 0};
char* uncompressed = 0;
if (isCompressed)
{
uncompressed = (char*)malloc(actualSize-8);
data2.cur = data2.start = uncompressed;
data2.end = uncompressed+actualSize-8;
uLongf uncompressedSize = actualSize-8;
uncompress((Bytef*)uncompressed, &uncompressedSize, data.as<Bytef>(), size-8);
}
else if (isCompressed)
{
data2.cur = data2.start = data.as<char>();
data2.end = data2.start+actualSize-8;
}
u8 bits = data2.read<u8>();
u8 numBits = bits>>3;
u32 rectSizeMinusOne = (numBits*4+5)>>3;
data2.move(rectSizeMinusOne);
const u16 frameRate = data2.read<u16>();
const u16 frameCount = data2.read<u16>();
std::set<u32> tagsUsed;
size_t tagCount = 0;
while (data2.cur!=data2.end)
{
u16 tagHeader = data2.read<u16>();
u32 tagLength = tagHeader&0x3F;
u32 tagType = tagHeader>>6;
tagsUsed.insert(tagType);
if (tagLength==0x3F)
tagLength = data2.read<u32>();
data2.move(tagLength);
parseTag(tagType);
++tagCount;
}
if (uncompressed) free(uncompressed);
printf("\nProcessed %d tags\n\n", tagCount);
printf(" Tags used \n");
printf("-------------------------\n");
std::set<u32>::iterator it = tagsUsed.begin(),
end = tagsUsed.end();
for (; it!=end; ++it)
{
parseTag(*it);
}
free(fileData);
}
/*
** batch doubling from [MSK15]
*/
__inline__ static void mul2_PIPE(__m128i *dat) {
unsigned a, b;
block tmp = le(dat[0]);
block carry[PIPE];
block up4, up8;
const block sh1 = _mm_set_epi8(255, 255, 255, 255, 255, 255, 15, 14, 255, 255, 255, 255, 255, 255, 7, 6);
const block sh2 = _mm_set_epi8(255, 255, 255, 255, 255, 255, 13, 12, 255, 255, 255, 255, 255, 255, 5, 4);
const block sh3 = _mm_set_epi8(255, 255, 255, 255, 255, 255, 11, 10, 255, 255, 255, 255, 255, 255, 3, 2);
const block sh4 = _mm_set_epi8(255, 255, 255, 255, 255, 255, 9, 8, 255, 255, 255, 255, 255, 255, 1, 0);
const block *Txp = (const block*)TX;
const block *Typ = (const block*)TY;
a = _mm_extract_epi8(tmp, 15);
b = _mm_extract_epi8(tmp, 7);
up4 = _mm_unpacklo_epi64(Txp[a], Typ[b]);
up8 = _mm_unpackhi_epi64(Txp[a], Typ[b]);
carry[0] = _mm_shuffle_epi8(up4, sh1);
carry[1] = _mm_shuffle_epi8(up4, sh2);
carry[2] = _mm_shuffle_epi8(up4, sh3);
carry[3] = _mm_shuffle_epi8(up4, sh4);
#if(PIPE>=5)
carry[4] = _mm_shuffle_epi8(up8, sh1);
#endif
#if(PIPE>=6)
carry[5] = _mm_shuffle_epi8(up8, sh2);
#endif
#if(PIPE>=7)
carry[6] = _mm_shuffle_epi8(up8, sh3);
#endif
#if(PIPE==8)
carry[7] = _mm_shuffle_epi8(up8, sh4);
#endif
dat[1] = _mm_slli_epi64(tmp, 1);
dat[2] = _mm_slli_epi64(tmp, 2);
dat[3] = _mm_slli_epi64(tmp, 3);
dat[4] = _mm_slli_epi64(tmp, 4);
#if(PIPE>=5)
dat[5] = _mm_slli_epi64(tmp, 5);
#endif
#if(PIPE>=6)
dat[6] = _mm_slli_epi64(tmp, 6);
#endif
#if(PIPE>=7)
dat[7] = _mm_slli_epi64(tmp, 7);
#endif
#if (PIPE==8)
dat[8] = (_mm_slli_epi64(tmp, 8));
#endif
dat[1] = le(_mm_xor_si128(dat[1], carry[0]));
dat[2] = le(_mm_xor_si128(dat[2], carry[1]));
dat[3] = le(_mm_xor_si128(dat[3], carry[2]));
dat[4] = le(_mm_xor_si128(dat[4], carry[3]));
#if(PIPE>=5)
dat[5] = le(_mm_xor_si128(dat[5], carry[4]));
#endif
#if(PIPE>=6)
dat[6] = le(_mm_xor_si128(dat[6], carry[5]));
#endif
#if(PIPE>=7)
dat[7] = le(_mm_xor_si128(dat[7], carry[6]));
#endif
#if (PIPE==8)
dat[8] = le(_mm_xor_si128(dat[8], carry[7]));
#endif
}
void aom_convolve8_add_src_hip_sse2(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4,
int w, int h) {
const int bd = 8;
assert(x_step_q4 == 16 && y_step_q4 == 16);
assert(!(w & 7));
(void)x_step_q4;
(void)y_step_q4;
uint16_t temp[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE];
int intermediate_height = h + SUBPEL_TAPS - 1;
int i, j;
const int center_tap = ((SUBPEL_TAPS - 1) / 2);
const uint8_t *const src_ptr = src - center_tap * src_stride - center_tap;
const __m128i zero = _mm_setzero_si128();
// Add an offset to account for the "add_src" part of the convolve function.
const __m128i offset = _mm_insert_epi16(zero, 1 << FILTER_BITS, 3);
/* Horizontal filter */
{
const __m128i coeffs_x =
_mm_add_epi16(_mm_loadu_si128((__m128i *)filter_x), offset);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(coeffs_x, coeffs_x);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(coeffs_x, coeffs_x);
// coeffs 0 1 0 1 0 1 0 1
const __m128i coeff_01 = _mm_unpacklo_epi64(tmp_0, tmp_0);
// coeffs 2 3 2 3 2 3 2 3
const __m128i coeff_23 = _mm_unpackhi_epi64(tmp_0, tmp_0);
// coeffs 4 5 4 5 4 5 4 5
const __m128i coeff_45 = _mm_unpacklo_epi64(tmp_1, tmp_1);
// coeffs 6 7 6 7 6 7 6 7
const __m128i coeff_67 = _mm_unpackhi_epi64(tmp_1, tmp_1);
const __m128i round_const =
_mm_set1_epi32((1 << (FILTER_BITS - EXTRAPREC_BITS - 1)) +
(1 << (bd + FILTER_BITS - 1)));
for (i = 0; i < intermediate_height; ++i) {
for (j = 0; j < w; j += 8) {
const __m128i data =
_mm_loadu_si128((__m128i *)&src_ptr[i * src_stride + j]);
// Filter even-index pixels
const __m128i src_0 = _mm_unpacklo_epi8(data, zero);
const __m128i res_0 = _mm_madd_epi16(src_0, coeff_01);
const __m128i src_2 = _mm_unpacklo_epi8(_mm_srli_si128(data, 2), zero);
const __m128i res_2 = _mm_madd_epi16(src_2, coeff_23);
const __m128i src_4 = _mm_unpacklo_epi8(_mm_srli_si128(data, 4), zero);
const __m128i res_4 = _mm_madd_epi16(src_4, coeff_45);
const __m128i src_6 = _mm_unpacklo_epi8(_mm_srli_si128(data, 6), zero);
const __m128i res_6 = _mm_madd_epi16(src_6, coeff_67);
__m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_4),
_mm_add_epi32(res_2, res_6));
res_even = _mm_srai_epi32(_mm_add_epi32(res_even, round_const),
FILTER_BITS - EXTRAPREC_BITS);
// Filter odd-index pixels
const __m128i src_1 = _mm_unpacklo_epi8(_mm_srli_si128(data, 1), zero);
const __m128i res_1 = _mm_madd_epi16(src_1, coeff_01);
const __m128i src_3 = _mm_unpacklo_epi8(_mm_srli_si128(data, 3), zero);
const __m128i res_3 = _mm_madd_epi16(src_3, coeff_23);
const __m128i src_5 = _mm_unpacklo_epi8(_mm_srli_si128(data, 5), zero);
const __m128i res_5 = _mm_madd_epi16(src_5, coeff_45);
const __m128i src_7 = _mm_unpacklo_epi8(_mm_srli_si128(data, 7), zero);
const __m128i res_7 = _mm_madd_epi16(src_7, coeff_67);
__m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_5),
_mm_add_epi32(res_3, res_7));
res_odd = _mm_srai_epi32(_mm_add_epi32(res_odd, round_const),
FILTER_BITS - EXTRAPREC_BITS);
// Pack in the column order 0, 2, 4, 6, 1, 3, 5, 7
__m128i res = _mm_packs_epi32(res_even, res_odd);
res = _mm_min_epi16(_mm_max_epi16(res, zero),
_mm_set1_epi16(EXTRAPREC_CLAMP_LIMIT(bd) - 1));
_mm_storeu_si128((__m128i *)&temp[i * MAX_SB_SIZE + j], res);
}
}
}
/* Vertical filter */
{
const __m128i coeffs_y =
_mm_add_epi16(_mm_loadu_si128((__m128i *)filter_y), offset);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(coeffs_y, coeffs_y);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(coeffs_y, coeffs_y);
// coeffs 0 1 0 1 0 1 0 1
const __m128i coeff_01 = _mm_unpacklo_epi64(tmp_0, tmp_0);
// coeffs 2 3 2 3 2 3 2 3
const __m128i coeff_23 = _mm_unpackhi_epi64(tmp_0, tmp_0);
// coeffs 4 5 4 5 4 5 4 5
const __m128i coeff_45 = _mm_unpacklo_epi64(tmp_1, tmp_1);
// coeffs 6 7 6 7 6 7 6 7
const __m128i coeff_67 = _mm_unpackhi_epi64(tmp_1, tmp_1);
const __m128i round_const =
_mm_set1_epi32((1 << (FILTER_BITS + EXTRAPREC_BITS - 1)) -
(1 << (bd + FILTER_BITS + EXTRAPREC_BITS - 1)));
for (i = 0; i < h; ++i) {
for (j = 0; j < w; j += 8) {
// Filter even-index pixels
const uint16_t *data = &temp[i * MAX_SB_SIZE + j];
const __m128i src_0 =
_mm_unpacklo_epi16(*(__m128i *)(data + 0 * MAX_SB_SIZE),
*(__m128i *)(data + 1 * MAX_SB_SIZE));
const __m128i src_2 =
_mm_unpacklo_epi16(*(__m128i *)(data + 2 * MAX_SB_SIZE),
*(__m128i *)(data + 3 * MAX_SB_SIZE));
const __m128i src_4 =
_mm_unpacklo_epi16(*(__m128i *)(data + 4 * MAX_SB_SIZE),
*(__m128i *)(data + 5 * MAX_SB_SIZE));
const __m128i src_6 =
_mm_unpacklo_epi16(*(__m128i *)(data + 6 * MAX_SB_SIZE),
*(__m128i *)(data + 7 * MAX_SB_SIZE));
const __m128i res_0 = _mm_madd_epi16(src_0, coeff_01);
const __m128i res_2 = _mm_madd_epi16(src_2, coeff_23);
const __m128i res_4 = _mm_madd_epi16(src_4, coeff_45);
const __m128i res_6 = _mm_madd_epi16(src_6, coeff_67);
const __m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_2),
_mm_add_epi32(res_4, res_6));
// Filter odd-index pixels
const __m128i src_1 =
_mm_unpackhi_epi16(*(__m128i *)(data + 0 * MAX_SB_SIZE),
*(__m128i *)(data + 1 * MAX_SB_SIZE));
const __m128i src_3 =
_mm_unpackhi_epi16(*(__m128i *)(data + 2 * MAX_SB_SIZE),
*(__m128i *)(data + 3 * MAX_SB_SIZE));
const __m128i src_5 =
_mm_unpackhi_epi16(*(__m128i *)(data + 4 * MAX_SB_SIZE),
*(__m128i *)(data + 5 * MAX_SB_SIZE));
const __m128i src_7 =
_mm_unpackhi_epi16(*(__m128i *)(data + 6 * MAX_SB_SIZE),
*(__m128i *)(data + 7 * MAX_SB_SIZE));
const __m128i res_1 = _mm_madd_epi16(src_1, coeff_01);
const __m128i res_3 = _mm_madd_epi16(src_3, coeff_23);
const __m128i res_5 = _mm_madd_epi16(src_5, coeff_45);
const __m128i res_7 = _mm_madd_epi16(src_7, coeff_67);
const __m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_3),
_mm_add_epi32(res_5, res_7));
// Rearrange pixels back into the order 0 ... 7
const __m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd);
const __m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd);
const __m128i res_lo_round = _mm_srai_epi32(
_mm_add_epi32(res_lo, round_const), FILTER_BITS + EXTRAPREC_BITS);
const __m128i res_hi_round = _mm_srai_epi32(
_mm_add_epi32(res_hi, round_const), FILTER_BITS + EXTRAPREC_BITS);
const __m128i res_16bit = _mm_packs_epi32(res_lo_round, res_hi_round);
__m128i res_8bit = _mm_packus_epi16(res_16bit, res_16bit);
__m128i *const p = (__m128i *)&dst[i * dst_stride + j];
_mm_storel_epi64(p, res_8bit);
}
}
}
}
void transpose_simd_8(const fhd_image* src, fhd_image* dst) {
assert(src->width == dst->height && src->height == dst->width);
assert(src->width % 8 == 0 && src->height % 8 == 0 && dst->width % 8 == 0 &&
dst->height % 8 == 0);
const int src_stride = src->width;
const int dst_stride = dst->width;
const uint16_t* src_data = src->data;
uint16_t* dst_data = dst->data;
for (int y = 0; y < src->height; y += 8) {
for (int x = 0; x < src_stride; x += 8) {
const int src_idx = y * src->width + x;
const int dst_idx = x * dst->width + y;
__m128i r0 = _mm_load_si128((const __m128i*)&src_data[src_idx]);
__m128i r1 =
_mm_load_si128((const __m128i*)&src_data[src_idx + src_stride]);
__m128i r2 =
_mm_load_si128((const __m128i*)&src_data[src_idx + src_stride * 2]);
__m128i r3 =
_mm_load_si128((const __m128i*)&src_data[src_idx + src_stride * 3]);
__m128i r4 =
_mm_load_si128((const __m128i*)&src_data[src_idx + src_stride * 4]);
__m128i r5 =
_mm_load_si128((const __m128i*)&src_data[src_idx + src_stride * 5]);
__m128i r6 =
_mm_load_si128((const __m128i*)&src_data[src_idx + src_stride * 6]);
__m128i r7 =
_mm_load_si128((const __m128i*)&src_data[src_idx + src_stride * 7]);
__m128i v0 = _mm_unpacklo_epi16(r0, r1);
__m128i v1 = _mm_unpacklo_epi16(r2, r3);
__m128i v2 = _mm_unpacklo_epi16(r4, r5);
__m128i v3 = _mm_unpacklo_epi16(r6, r7);
__m128i v4 = _mm_unpackhi_epi16(r0, r1);
__m128i v5 = _mm_unpackhi_epi16(r2, r3);
__m128i v6 = _mm_unpackhi_epi16(r4, r5);
__m128i v7 = _mm_unpackhi_epi16(r6, r7);
__m128i c0 = _mm_unpacklo_epi32(v0, v1);
__m128i c1 = _mm_unpackhi_epi32(v0, v1);
__m128i c2 = _mm_unpacklo_epi32(v2, v3);
__m128i c3 = _mm_unpackhi_epi32(v2, v3);
__m128i c4 = _mm_unpacklo_epi32(v4, v5);
__m128i c5 = _mm_unpackhi_epi32(v4, v5);
__m128i c6 = _mm_unpacklo_epi32(v6, v7);
__m128i c7 = _mm_unpackhi_epi32(v6, v7);
__m128i d0 = _mm_unpacklo_epi64(c0, c2);
__m128i d1 = _mm_unpackhi_epi64(c0, c2);
__m128i d2 = _mm_unpacklo_epi64(c1, c3);
__m128i d3 = _mm_unpackhi_epi64(c1, c3);
__m128i d4 = _mm_unpacklo_epi64(c4, c6);
__m128i d5 = _mm_unpackhi_epi64(c4, c6);
__m128i d6 = _mm_unpacklo_epi64(c5, c7);
__m128i d7 = _mm_unpackhi_epi64(c5, c7);
_mm_store_si128((__m128i*)&dst_data[dst_idx], d0);
_mm_store_si128((__m128i*)&dst_data[dst_idx + dst_stride], d1);
_mm_store_si128((__m128i*)&dst_data[dst_idx + dst_stride * 2], d2);
_mm_store_si128((__m128i*)&dst_data[dst_idx + dst_stride * 3], d3);
_mm_store_si128((__m128i*)&dst_data[dst_idx + dst_stride * 4], d4);
_mm_store_si128((__m128i*)&dst_data[dst_idx + dst_stride * 5], d5);
_mm_store_si128((__m128i*)&dst_data[dst_idx + dst_stride * 6], d6);
_mm_store_si128((__m128i*)&dst_data[dst_idx + dst_stride * 7], d7);
}
}
}
// Hadamard transform
// Returns the difference between the weighted sum of the absolute value of
// transformed coefficients.
static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB,
const uint16_t* const w) {
int32_t sum[4];
__m128i tmp_0, tmp_1, tmp_2, tmp_3;
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m128i three = _mm_set1_epi16(3);
// Load, combine and tranpose inputs.
{
const __m128i inA_0 = _mm_loadl_epi64((__m128i*)&inA[BPS * 0]);
const __m128i inA_1 = _mm_loadl_epi64((__m128i*)&inA[BPS * 1]);
const __m128i inA_2 = _mm_loadl_epi64((__m128i*)&inA[BPS * 2]);
const __m128i inA_3 = _mm_loadl_epi64((__m128i*)&inA[BPS * 3]);
const __m128i inB_0 = _mm_loadl_epi64((__m128i*)&inB[BPS * 0]);
const __m128i inB_1 = _mm_loadl_epi64((__m128i*)&inB[BPS * 1]);
const __m128i inB_2 = _mm_loadl_epi64((__m128i*)&inB[BPS * 2]);
const __m128i inB_3 = _mm_loadl_epi64((__m128i*)&inB[BPS * 3]);
// Combine inA and inB (we'll do two transforms in parallel).
const __m128i inAB_0 = _mm_unpacklo_epi8(inA_0, inB_0);
const __m128i inAB_1 = _mm_unpacklo_epi8(inA_1, inB_1);
const __m128i inAB_2 = _mm_unpacklo_epi8(inA_2, inB_2);
const __m128i inAB_3 = _mm_unpacklo_epi8(inA_3, inB_3);
// a00 b00 a01 b01 a02 b03 a03 b03 0 0 0 0 0 0 0 0
// a10 b10 a11 b11 a12 b12 a13 b13 0 0 0 0 0 0 0 0
// a20 b20 a21 b21 a22 b22 a23 b23 0 0 0 0 0 0 0 0
// a30 b30 a31 b31 a32 b32 a33 b33 0 0 0 0 0 0 0 0
// Transpose the two 4x4, discarding the filling zeroes.
const __m128i transpose0_0 = _mm_unpacklo_epi8(inAB_0, inAB_2);
const __m128i transpose0_1 = _mm_unpacklo_epi8(inAB_1, inAB_3);
// a00 a20 b00 b20 a01 a21 b01 b21 a02 a22 b02 b22 a03 a23 b03 b23
// a10 a30 b10 b30 a11 a31 b11 b31 a12 a32 b12 b32 a13 a33 b13 b33
const __m128i transpose1_0 = _mm_unpacklo_epi8(transpose0_0, transpose0_1);
const __m128i transpose1_1 = _mm_unpackhi_epi8(transpose0_0, transpose0_1);
// a00 a10 a20 a30 b00 b10 b20 b30 a01 a11 a21 a31 b01 b11 b21 b31
// a02 a12 a22 a32 b02 b12 b22 b32 a03 a13 a23 a33 b03 b13 b23 b33
// Convert to 16b.
tmp_0 = _mm_unpacklo_epi8(transpose1_0, zero);
tmp_1 = _mm_unpackhi_epi8(transpose1_0, zero);
tmp_2 = _mm_unpacklo_epi8(transpose1_1, zero);
tmp_3 = _mm_unpackhi_epi8(transpose1_1, zero);
// a00 a10 a20 a30 b00 b10 b20 b30
// a01 a11 a21 a31 b01 b11 b21 b31
// a02 a12 a22 a32 b02 b12 b22 b32
// a03 a13 a23 a33 b03 b13 b23 b33
}
// Horizontal pass and subsequent transpose.
{
// Calculate a and b (two 4x4 at once).
const __m128i a0 = _mm_slli_epi16(_mm_add_epi16(tmp_0, tmp_2), 2);
const __m128i a1 = _mm_slli_epi16(_mm_add_epi16(tmp_1, tmp_3), 2);
const __m128i a2 = _mm_slli_epi16(_mm_sub_epi16(tmp_1, tmp_3), 2);
const __m128i a3 = _mm_slli_epi16(_mm_sub_epi16(tmp_0, tmp_2), 2);
// b0_extra = (a0 != 0);
const __m128i b0_extra = _mm_andnot_si128(_mm_cmpeq_epi16 (a0, zero), one);
const __m128i b0_base = _mm_add_epi16(a0, a1);
const __m128i b1 = _mm_add_epi16(a3, a2);
const __m128i b2 = _mm_sub_epi16(a3, a2);
const __m128i b3 = _mm_sub_epi16(a0, a1);
const __m128i b0 = _mm_add_epi16(b0_base, b0_extra);
// a00 a01 a02 a03 b00 b01 b02 b03
// a10 a11 a12 a13 b10 b11 b12 b13
// a20 a21 a22 a23 b20 b21 b22 b23
// a30 a31 a32 a33 b30 b31 b32 b33
// Transpose the two 4x4.
const __m128i transpose0_0 = _mm_unpacklo_epi16(b0, b1);
const __m128i transpose0_1 = _mm_unpacklo_epi16(b2, b3);
const __m128i transpose0_2 = _mm_unpackhi_epi16(b0, b1);
const __m128i transpose0_3 = _mm_unpackhi_epi16(b2, b3);
// a00 a10 a01 a11 a02 a12 a03 a13
// a20 a30 a21 a31 a22 a32 a23 a33
// b00 b10 b01 b11 b02 b12 b03 b13
// b20 b30 b21 b31 b22 b32 b23 b33
const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1);
const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3);
const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3);
// a00 a10 a20 a30 a01 a11 a21 a31
// b00 b10 b20 b30 b01 b11 b21 b31
// a02 a12 a22 a32 a03 a13 a23 a33
// b02 b12 a22 b32 b03 b13 b23 b33
tmp_0 = _mm_unpacklo_epi64(transpose1_0, transpose1_1);
tmp_1 = _mm_unpackhi_epi64(transpose1_0, transpose1_1);
tmp_2 = _mm_unpacklo_epi64(transpose1_2, transpose1_3);
tmp_3 = _mm_unpackhi_epi64(transpose1_2, transpose1_3);
// a00 a10 a20 a30 b00 b10 b20 b30
// a01 a11 a21 a31 b01 b11 b21 b31
// a02 a12 a22 a32 b02 b12 b22 b32
// a03 a13 a23 a33 b03 b13 b23 b33
}
// Vertical pass and difference of weighted sums.
{
// Load all inputs.
// TODO(cduvivier): Make variable declarations and allocations aligned so
// we can use _mm_load_si128 instead of _mm_loadu_si128.
const __m128i w_0 = _mm_loadu_si128((__m128i*)&w[0]);
const __m128i w_8 = _mm_loadu_si128((__m128i*)&w[8]);
// Calculate a and b (two 4x4 at once).
const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2);
const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3);
const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3);
const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2);
const __m128i b0 = _mm_add_epi16(a0, a1);
const __m128i b1 = _mm_add_epi16(a3, a2);
const __m128i b2 = _mm_sub_epi16(a3, a2);
const __m128i b3 = _mm_sub_epi16(a0, a1);
// Separate the transforms of inA and inB.
__m128i A_b0 = _mm_unpacklo_epi64(b0, b1);
__m128i A_b2 = _mm_unpacklo_epi64(b2, b3);
__m128i B_b0 = _mm_unpackhi_epi64(b0, b1);
__m128i B_b2 = _mm_unpackhi_epi64(b2, b3);
{
// sign(b) = b >> 15 (0x0000 if positive, 0xffff if negative)
const __m128i sign_A_b0 = _mm_srai_epi16(A_b0, 15);
const __m128i sign_A_b2 = _mm_srai_epi16(A_b2, 15);
const __m128i sign_B_b0 = _mm_srai_epi16(B_b0, 15);
const __m128i sign_B_b2 = _mm_srai_epi16(B_b2, 15);
// b = abs(b) = (b ^ sign) - sign
A_b0 = _mm_xor_si128(A_b0, sign_A_b0);
A_b2 = _mm_xor_si128(A_b2, sign_A_b2);
B_b0 = _mm_xor_si128(B_b0, sign_B_b0);
B_b2 = _mm_xor_si128(B_b2, sign_B_b2);
A_b0 = _mm_sub_epi16(A_b0, sign_A_b0);
A_b2 = _mm_sub_epi16(A_b2, sign_A_b2);
B_b0 = _mm_sub_epi16(B_b0, sign_B_b0);
B_b2 = _mm_sub_epi16(B_b2, sign_B_b2);
}
// b = abs(b) + 3
A_b0 = _mm_add_epi16(A_b0, three);
A_b2 = _mm_add_epi16(A_b2, three);
B_b0 = _mm_add_epi16(B_b0, three);
B_b2 = _mm_add_epi16(B_b2, three);
// abs((b + (b<0) + 3) >> 3) = (abs(b) + 3) >> 3
// b = (abs(b) + 3) >> 3
A_b0 = _mm_srai_epi16(A_b0, 3);
A_b2 = _mm_srai_epi16(A_b2, 3);
B_b0 = _mm_srai_epi16(B_b0, 3);
B_b2 = _mm_srai_epi16(B_b2, 3);
// weighted sums
A_b0 = _mm_madd_epi16(A_b0, w_0);
A_b2 = _mm_madd_epi16(A_b2, w_8);
B_b0 = _mm_madd_epi16(B_b0, w_0);
B_b2 = _mm_madd_epi16(B_b2, w_8);
A_b0 = _mm_add_epi32(A_b0, A_b2);
B_b0 = _mm_add_epi32(B_b0, B_b2);
// difference of weighted sums
A_b0 = _mm_sub_epi32(A_b0, B_b0);
_mm_storeu_si128((__m128i*)&sum[0], A_b0);
}
return sum[0] + sum[1] + sum[2] + sum[3];
}
void vp9_fdct8x8_quant_ssse3(
const int16_t *input, int stride, tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr, const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
uint16_t *eob_ptr, const int16_t *scan_ptr, const int16_t *iscan_ptr) {
__m128i zero;
int pass;
// Constants
// When we use them, in one case, they are all the same. In all others
// it's a pair of them that we need to repeat four times. This is done
// by constructing the 32 bit constant corresponding to that pair.
const __m128i k__dual_p16_p16 = dual_set_epi16(23170, 23170);
const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64);
const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
// Load input
__m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
__m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
__m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
__m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
__m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride));
__m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride));
__m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride));
__m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride));
__m128i *in[8];
int index = 0;
(void)scan_ptr;
(void)zbin_ptr;
(void)quant_shift_ptr;
(void)coeff_ptr;
// Pre-condition input (shift by two)
in0 = _mm_slli_epi16(in0, 2);
in1 = _mm_slli_epi16(in1, 2);
in2 = _mm_slli_epi16(in2, 2);
in3 = _mm_slli_epi16(in3, 2);
in4 = _mm_slli_epi16(in4, 2);
in5 = _mm_slli_epi16(in5, 2);
in6 = _mm_slli_epi16(in6, 2);
in7 = _mm_slli_epi16(in7, 2);
in[0] = &in0;
in[1] = &in1;
in[2] = &in2;
in[3] = &in3;
in[4] = &in4;
in[5] = &in5;
in[6] = &in6;
in[7] = &in7;
// We do two passes, first the columns, then the rows. The results of the
// first pass are transposed so that the same column code can be reused. The
// results of the second pass are also transposed so that the rows (processed
// as columns) are put back in row positions.
for (pass = 0; pass < 2; pass++) {
// To store results of each pass before the transpose.
__m128i res0, res1, res2, res3, res4, res5, res6, res7;
// Add/subtract
const __m128i q0 = _mm_add_epi16(in0, in7);
const __m128i q1 = _mm_add_epi16(in1, in6);
const __m128i q2 = _mm_add_epi16(in2, in5);
const __m128i q3 = _mm_add_epi16(in3, in4);
const __m128i q4 = _mm_sub_epi16(in3, in4);
const __m128i q5 = _mm_sub_epi16(in2, in5);
const __m128i q6 = _mm_sub_epi16(in1, in6);
const __m128i q7 = _mm_sub_epi16(in0, in7);
// Work on first four results
{
// Add/subtract
const __m128i r0 = _mm_add_epi16(q0, q3);
const __m128i r1 = _mm_add_epi16(q1, q2);
const __m128i r2 = _mm_sub_epi16(q1, q2);
const __m128i r3 = _mm_sub_epi16(q0, q3);
// Interleave to do the multiply by constants which gets us into 32bits
const __m128i t0 = _mm_unpacklo_epi16(r0, r1);
const __m128i t1 = _mm_unpackhi_epi16(r0, r1);
const __m128i t2 = _mm_unpacklo_epi16(r2, r3);
const __m128i t3 = _mm_unpackhi_epi16(r2, r3);
const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16);
const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16);
const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16);
const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08);
const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08);
const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24);
const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24);
// dct_const_round_shift
const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
// Combine
res0 = _mm_packs_epi32(w0, w1);
res4 = _mm_packs_epi32(w2, w3);
res2 = _mm_packs_epi32(w4, w5);
res6 = _mm_packs_epi32(w6, w7);
}
// Work on next four results
{
// Interleave to do the multiply by constants which gets us into 32bits
const __m128i d0 = _mm_sub_epi16(q6, q5);
const __m128i d1 = _mm_add_epi16(q6, q5);
const __m128i r0 = _mm_mulhrs_epi16(d0, k__dual_p16_p16);
const __m128i r1 = _mm_mulhrs_epi16(d1, k__dual_p16_p16);
// Add/subtract
const __m128i x0 = _mm_add_epi16(q4, r0);
const __m128i x1 = _mm_sub_epi16(q4, r0);
const __m128i x2 = _mm_sub_epi16(q7, r1);
const __m128i x3 = _mm_add_epi16(q7, r1);
// Interleave to do the multiply by constants which gets us into 32bits
const __m128i t0 = _mm_unpacklo_epi16(x0, x3);
const __m128i t1 = _mm_unpackhi_epi16(x0, x3);
const __m128i t2 = _mm_unpacklo_epi16(x1, x2);
const __m128i t3 = _mm_unpackhi_epi16(x1, x2);
const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04);
const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04);
const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28);
const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28);
const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20);
const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20);
const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12);
const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12);
// dct_const_round_shift
const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
// Combine
res1 = _mm_packs_epi32(w0, w1);
res7 = _mm_packs_epi32(w2, w3);
res5 = _mm_packs_epi32(w4, w5);
res3 = _mm_packs_epi32(w6, w7);
}
// Transpose the 8x8.
{
// 00 01 02 03 04 05 06 07
// 10 11 12 13 14 15 16 17
// 20 21 22 23 24 25 26 27
// 30 31 32 33 34 35 36 37
// 40 41 42 43 44 45 46 47
// 50 51 52 53 54 55 56 57
// 60 61 62 63 64 65 66 67
// 70 71 72 73 74 75 76 77
const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1);
const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3);
const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1);
const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3);
const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5);
const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7);
const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5);
const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7);
// 00 10 01 11 02 12 03 13
// 20 30 21 31 22 32 23 33
// 04 14 05 15 06 16 07 17
// 24 34 25 35 26 36 27 37
// 40 50 41 51 42 52 43 53
// 60 70 61 71 62 72 63 73
// 54 54 55 55 56 56 57 57
// 64 74 65 75 66 76 67 77
const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3);
const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3);
const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5);
const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7);
const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5);
const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7);
// 00 10 20 30 01 11 21 31
// 40 50 60 70 41 51 61 71
// 02 12 22 32 03 13 23 33
// 42 52 62 72 43 53 63 73
// 04 14 24 34 05 15 21 36
// 44 54 64 74 45 55 61 76
// 06 16 26 36 07 17 27 37
// 46 56 66 76 47 57 67 77
in0 = _mm_unpacklo_epi64(tr1_0, tr1_4);
in1 = _mm_unpackhi_epi64(tr1_0, tr1_4);
in2 = _mm_unpacklo_epi64(tr1_2, tr1_6);
in3 = _mm_unpackhi_epi64(tr1_2, tr1_6);
in4 = _mm_unpacklo_epi64(tr1_1, tr1_5);
in5 = _mm_unpackhi_epi64(tr1_1, tr1_5);
in6 = _mm_unpacklo_epi64(tr1_3, tr1_7);
in7 = _mm_unpackhi_epi64(tr1_3, tr1_7);
// 00 10 20 30 40 50 60 70
// 01 11 21 31 41 51 61 71
// 02 12 22 32 42 52 62 72
// 03 13 23 33 43 53 63 73
// 04 14 24 34 44 54 64 74
// 05 15 25 35 45 55 65 75
// 06 16 26 36 46 56 66 76
// 07 17 27 37 47 57 67 77
}
}
// Post-condition output and store it
{
// Post-condition (division by two)
// division of two 16 bits signed numbers using shifts
// n / 2 = (n - (n >> 15)) >> 1
const __m128i sign_in0 = _mm_srai_epi16(in0, 15);
const __m128i sign_in1 = _mm_srai_epi16(in1, 15);
const __m128i sign_in2 = _mm_srai_epi16(in2, 15);
const __m128i sign_in3 = _mm_srai_epi16(in3, 15);
const __m128i sign_in4 = _mm_srai_epi16(in4, 15);
const __m128i sign_in5 = _mm_srai_epi16(in5, 15);
const __m128i sign_in6 = _mm_srai_epi16(in6, 15);
const __m128i sign_in7 = _mm_srai_epi16(in7, 15);
in0 = _mm_sub_epi16(in0, sign_in0);
in1 = _mm_sub_epi16(in1, sign_in1);
in2 = _mm_sub_epi16(in2, sign_in2);
in3 = _mm_sub_epi16(in3, sign_in3);
in4 = _mm_sub_epi16(in4, sign_in4);
in5 = _mm_sub_epi16(in5, sign_in5);
in6 = _mm_sub_epi16(in6, sign_in6);
in7 = _mm_sub_epi16(in7, sign_in7);
in0 = _mm_srai_epi16(in0, 1);
in1 = _mm_srai_epi16(in1, 1);
in2 = _mm_srai_epi16(in2, 1);
in3 = _mm_srai_epi16(in3, 1);
in4 = _mm_srai_epi16(in4, 1);
in5 = _mm_srai_epi16(in5, 1);
in6 = _mm_srai_epi16(in6, 1);
in7 = _mm_srai_epi16(in7, 1);
}
iscan_ptr += n_coeffs;
qcoeff_ptr += n_coeffs;
dqcoeff_ptr += n_coeffs;
n_coeffs = -n_coeffs;
zero = _mm_setzero_si128();
if (!skip_block) {
__m128i eob;
__m128i round, quant, dequant, thr;
int16_t nzflag;
{
__m128i coeff0, coeff1;
// Setup global values
{
round = _mm_load_si128((const __m128i *)round_ptr);
quant = _mm_load_si128((const __m128i *)quant_ptr);
dequant = _mm_load_si128((const __m128i *)dequant_ptr);
}
{
__m128i coeff0_sign, coeff1_sign;
__m128i qcoeff0, qcoeff1;
__m128i qtmp0, qtmp1;
// Do DC and first 15 AC
coeff0 = *in[0];
coeff1 = *in[1];
// Poor man's sign extract
coeff0_sign = _mm_srai_epi16(coeff0, 15);
coeff1_sign = _mm_srai_epi16(coeff1, 15);
qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign);
qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign);
qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign);
qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign);
qcoeff0 = _mm_adds_epi16(qcoeff0, round);
round = _mm_unpackhi_epi64(round, round);
qcoeff1 = _mm_adds_epi16(qcoeff1, round);
qtmp0 = _mm_mulhi_epi16(qcoeff0, quant);
quant = _mm_unpackhi_epi64(quant, quant);
qtmp1 = _mm_mulhi_epi16(qcoeff1, quant);
// Reinsert signs
qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign);
qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign);
qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign);
qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign);
store_tran_low(qcoeff0, qcoeff_ptr + n_coeffs);
store_tran_low(qcoeff1, qcoeff_ptr + n_coeffs + 8);
coeff0 = _mm_mullo_epi16(qcoeff0, dequant);
dequant = _mm_unpackhi_epi64(dequant, dequant);
coeff1 = _mm_mullo_epi16(qcoeff1, dequant);
store_tran_low(coeff0, dqcoeff_ptr + n_coeffs);
store_tran_low(coeff1, dqcoeff_ptr + n_coeffs + 8);
}
{
// Scan for eob
__m128i zero_coeff0, zero_coeff1;
__m128i nzero_coeff0, nzero_coeff1;
__m128i iscan0, iscan1;
__m128i eob1;
zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero);
zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero);
nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero);
nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero);
iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs));
iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1);
// Add one to convert from indices to counts
iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0);
iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1);
eob = _mm_and_si128(iscan0, nzero_coeff0);
eob1 = _mm_and_si128(iscan1, nzero_coeff1);
eob = _mm_max_epi16(eob, eob1);
}
n_coeffs += 8 * 2;
}
// AC only loop
index = 2;
thr = _mm_srai_epi16(dequant, 1);
while (n_coeffs < 0) {
__m128i coeff0, coeff1;
{
__m128i coeff0_sign, coeff1_sign;
__m128i qcoeff0, qcoeff1;
__m128i qtmp0, qtmp1;
assert(index < (int)(sizeof(in) / sizeof(in[0])) - 1);
coeff0 = *in[index];
coeff1 = *in[index + 1];
// Poor man's sign extract
coeff0_sign = _mm_srai_epi16(coeff0, 15);
coeff1_sign = _mm_srai_epi16(coeff1, 15);
qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign);
qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign);
qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign);
qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign);
nzflag = _mm_movemask_epi8(_mm_cmpgt_epi16(qcoeff0, thr)) |
_mm_movemask_epi8(_mm_cmpgt_epi16(qcoeff1, thr));
if (nzflag) {
qcoeff0 = _mm_adds_epi16(qcoeff0, round);
qcoeff1 = _mm_adds_epi16(qcoeff1, round);
qtmp0 = _mm_mulhi_epi16(qcoeff0, quant);
qtmp1 = _mm_mulhi_epi16(qcoeff1, quant);
// Reinsert signs
qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign);
qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign);
qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign);
qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign);
store_tran_low(qcoeff0, qcoeff_ptr + n_coeffs);
store_tran_low(qcoeff1, qcoeff_ptr + n_coeffs + 8);
coeff0 = _mm_mullo_epi16(qcoeff0, dequant);
coeff1 = _mm_mullo_epi16(qcoeff1, dequant);
store_tran_low(coeff0, dqcoeff_ptr + n_coeffs);
store_tran_low(coeff1, dqcoeff_ptr + n_coeffs + 8);
} else {
// Maybe a more efficient way to store 0?
store_zero_tran_low(qcoeff_ptr + n_coeffs);
store_zero_tran_low(qcoeff_ptr + n_coeffs + 8);
store_zero_tran_low(dqcoeff_ptr + n_coeffs);
store_zero_tran_low(dqcoeff_ptr + n_coeffs + 8);
}
}
if (nzflag) {
// Scan for eob
__m128i zero_coeff0, zero_coeff1;
__m128i nzero_coeff0, nzero_coeff1;
__m128i iscan0, iscan1;
__m128i eob0, eob1;
zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero);
zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero);
nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero);
nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero);
iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs));
iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1);
// Add one to convert from indices to counts
iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0);
iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1);
eob0 = _mm_and_si128(iscan0, nzero_coeff0);
eob1 = _mm_and_si128(iscan1, nzero_coeff1);
eob0 = _mm_max_epi16(eob0, eob1);
eob = _mm_max_epi16(eob, eob0);
}
n_coeffs += 8 * 2;
index += 2;
}
// Accumulate EOB
{
__m128i eob_shuffled;
eob_shuffled = _mm_shuffle_epi32(eob, 0xe);
eob = _mm_max_epi16(eob, eob_shuffled);
eob_shuffled = _mm_shufflelo_epi16(eob, 0xe);
eob = _mm_max_epi16(eob, eob_shuffled);
eob_shuffled = _mm_shufflelo_epi16(eob, 0x1);
eob = _mm_max_epi16(eob, eob_shuffled);
*eob_ptr = _mm_extract_epi16(eob, 1);
}
} else {
do {
store_zero_tran_low(dqcoeff_ptr + n_coeffs);
store_zero_tran_low(dqcoeff_ptr + n_coeffs + 8);
store_zero_tran_low(qcoeff_ptr + n_coeffs);
store_zero_tran_low(qcoeff_ptr + n_coeffs + 8);
n_coeffs += 8 * 2;
} while (n_coeffs < 0);
*eob_ptr = 0;
}
}
opus_val32 celt_inner_prod_sse4_1(const opus_val16 *x, const opus_val16 *y,
int N)
{
opus_int i, dataSize16;
opus_int32 sum;
__m128i inVec1_76543210, inVec1_FEDCBA98, acc1;
__m128i inVec2_76543210, inVec2_FEDCBA98, acc2;
__m128i inVec1_3210, inVec2_3210;
sum = 0;
dataSize16 = N & ~15;
acc1 = _mm_setzero_si128();
acc2 = _mm_setzero_si128();
for (i=0;i<dataSize16;i+=16) {
inVec1_76543210 = _mm_loadu_si128((__m128i *)(&x[i + 0]));
inVec2_76543210 = _mm_loadu_si128((__m128i *)(&y[i + 0]));
inVec1_FEDCBA98 = _mm_loadu_si128((__m128i *)(&x[i + 8]));
inVec2_FEDCBA98 = _mm_loadu_si128((__m128i *)(&y[i + 8]));
inVec1_76543210 = _mm_madd_epi16(inVec1_76543210, inVec2_76543210);
inVec1_FEDCBA98 = _mm_madd_epi16(inVec1_FEDCBA98, inVec2_FEDCBA98);
acc1 = _mm_add_epi32(acc1, inVec1_76543210);
acc2 = _mm_add_epi32(acc2, inVec1_FEDCBA98);
}
acc1 = _mm_add_epi32(acc1, acc2);
if (N - i >= 8)
{
inVec1_76543210 = _mm_loadu_si128((__m128i *)(&x[i + 0]));
inVec2_76543210 = _mm_loadu_si128((__m128i *)(&y[i + 0]));
inVec1_76543210 = _mm_madd_epi16(inVec1_76543210, inVec2_76543210);
acc1 = _mm_add_epi32(acc1, inVec1_76543210);
i += 8;
}
if (N - i >= 4)
{
inVec1_3210 = OP_CVTEPI16_EPI32_M64(&x[i + 0]);
inVec2_3210 = OP_CVTEPI16_EPI32_M64(&y[i + 0]);
inVec1_3210 = _mm_mullo_epi32(inVec1_3210, inVec2_3210);
acc1 = _mm_add_epi32(acc1, inVec1_3210);
i += 4;
}
acc1 = _mm_add_epi32(acc1, _mm_unpackhi_epi64(acc1, acc1));
acc1 = _mm_add_epi32(acc1, _mm_shufflelo_epi16(acc1, 0x0E));
sum += _mm_cvtsi128_si32(acc1);
for (;i<N;i++)
{
sum = silk_SMLABB(sum, x[i], y[i]);
}
return sum;
}
static void FTransform(const uint8_t* src, const uint8_t* ref, int16_t* out) {
const __m128i zero = _mm_setzero_si128();
const __m128i seven = _mm_set1_epi16(7);
const __m128i k937 = _mm_set1_epi32(937);
const __m128i k1812 = _mm_set1_epi32(1812);
const __m128i k51000 = _mm_set1_epi32(51000);
const __m128i k12000_plus_one = _mm_set1_epi32(12000 + (1 << 16));
const __m128i k5352_2217 = _mm_set_epi16(5352, 2217, 5352, 2217,
5352, 2217, 5352, 2217);
const __m128i k2217_5352 = _mm_set_epi16(2217, -5352, 2217, -5352,
2217, -5352, 2217, -5352);
const __m128i k88p = _mm_set_epi16(8, 8, 8, 8, 8, 8, 8, 8);
const __m128i k88m = _mm_set_epi16(-8, 8, -8, 8, -8, 8, -8, 8);
const __m128i k5352_2217p = _mm_set_epi16(2217, 5352, 2217, 5352,
2217, 5352, 2217, 5352);
const __m128i k5352_2217m = _mm_set_epi16(-5352, 2217, -5352, 2217,
-5352, 2217, -5352, 2217);
__m128i v01, v32;
// Difference between src and ref and initial transpose.
{
// Load src and convert to 16b.
const __m128i src0 = _mm_loadl_epi64((const __m128i*)&src[0 * BPS]);
const __m128i src1 = _mm_loadl_epi64((const __m128i*)&src[1 * BPS]);
const __m128i src2 = _mm_loadl_epi64((const __m128i*)&src[2 * BPS]);
const __m128i src3 = _mm_loadl_epi64((const __m128i*)&src[3 * BPS]);
const __m128i src_0 = _mm_unpacklo_epi8(src0, zero);
const __m128i src_1 = _mm_unpacklo_epi8(src1, zero);
const __m128i src_2 = _mm_unpacklo_epi8(src2, zero);
const __m128i src_3 = _mm_unpacklo_epi8(src3, zero);
// Load ref and convert to 16b.
const __m128i ref0 = _mm_loadl_epi64((const __m128i*)&ref[0 * BPS]);
const __m128i ref1 = _mm_loadl_epi64((const __m128i*)&ref[1 * BPS]);
const __m128i ref2 = _mm_loadl_epi64((const __m128i*)&ref[2 * BPS]);
const __m128i ref3 = _mm_loadl_epi64((const __m128i*)&ref[3 * BPS]);
const __m128i ref_0 = _mm_unpacklo_epi8(ref0, zero);
const __m128i ref_1 = _mm_unpacklo_epi8(ref1, zero);
const __m128i ref_2 = _mm_unpacklo_epi8(ref2, zero);
const __m128i ref_3 = _mm_unpacklo_epi8(ref3, zero);
// Compute difference. -> 00 01 02 03 00 00 00 00
const __m128i diff0 = _mm_sub_epi16(src_0, ref_0);
const __m128i diff1 = _mm_sub_epi16(src_1, ref_1);
const __m128i diff2 = _mm_sub_epi16(src_2, ref_2);
const __m128i diff3 = _mm_sub_epi16(src_3, ref_3);
// Unpack and shuffle
// 00 01 02 03 0 0 0 0
// 10 11 12 13 0 0 0 0
// 20 21 22 23 0 0 0 0
// 30 31 32 33 0 0 0 0
const __m128i shuf01 = _mm_unpacklo_epi32(diff0, diff1);
const __m128i shuf23 = _mm_unpacklo_epi32(diff2, diff3);
// 00 01 10 11 02 03 12 13
// 20 21 30 31 22 23 32 33
const __m128i shuf01_p =
_mm_shufflehi_epi16(shuf01, _MM_SHUFFLE(2, 3, 0, 1));
const __m128i shuf23_p =
_mm_shufflehi_epi16(shuf23, _MM_SHUFFLE(2, 3, 0, 1));
// 00 01 10 11 03 02 13 12
// 20 21 30 31 23 22 33 32
const __m128i s01 = _mm_unpacklo_epi64(shuf01_p, shuf23_p);
const __m128i s32 = _mm_unpackhi_epi64(shuf01_p, shuf23_p);
// 00 01 10 11 20 21 30 31
// 03 02 13 12 23 22 33 32
const __m128i a01 = _mm_add_epi16(s01, s32);
const __m128i a32 = _mm_sub_epi16(s01, s32);
// [d0 + d3 | d1 + d2 | ...] = [a0 a1 | a0' a1' | ... ]
// [d0 - d3 | d1 - d2 | ...] = [a3 a2 | a3' a2' | ... ]
const __m128i tmp0 = _mm_madd_epi16(a01, k88p); // [ (a0 + a1) << 3, ... ]
const __m128i tmp2 = _mm_madd_epi16(a01, k88m); // [ (a0 - a1) << 3, ... ]
const __m128i tmp1_1 = _mm_madd_epi16(a32, k5352_2217p);
const __m128i tmp3_1 = _mm_madd_epi16(a32, k5352_2217m);
const __m128i tmp1_2 = _mm_add_epi32(tmp1_1, k1812);
const __m128i tmp3_2 = _mm_add_epi32(tmp3_1, k937);
const __m128i tmp1 = _mm_srai_epi32(tmp1_2, 9);
const __m128i tmp3 = _mm_srai_epi32(tmp3_2, 9);
const __m128i s03 = _mm_packs_epi32(tmp0, tmp2);
const __m128i s12 = _mm_packs_epi32(tmp1, tmp3);
const __m128i s_lo = _mm_unpacklo_epi16(s03, s12); // 0 1 0 1 0 1...
const __m128i s_hi = _mm_unpackhi_epi16(s03, s12); // 2 3 2 3 2 3
const __m128i v23 = _mm_unpackhi_epi32(s_lo, s_hi);
v01 = _mm_unpacklo_epi32(s_lo, s_hi);
v32 = _mm_shuffle_epi32(v23, _MM_SHUFFLE(1, 0, 3, 2)); // 3 2 3 2 3 2..
}
// Second pass
{
// Same operations are done on the (0,3) and (1,2) pairs.
// a0 = v0 + v3
// a1 = v1 + v2
// a3 = v0 - v3
// a2 = v1 - v2
const __m128i a01 = _mm_add_epi16(v01, v32);
const __m128i a32 = _mm_sub_epi16(v01, v32);
const __m128i a11 = _mm_unpackhi_epi64(a01, a01);
const __m128i a22 = _mm_unpackhi_epi64(a32, a32);
const __m128i a01_plus_7 = _mm_add_epi16(a01, seven);
// d0 = (a0 + a1 + 7) >> 4;
// d2 = (a0 - a1 + 7) >> 4;
const __m128i c0 = _mm_add_epi16(a01_plus_7, a11);
const __m128i c2 = _mm_sub_epi16(a01_plus_7, a11);
const __m128i d0 = _mm_srai_epi16(c0, 4);
const __m128i d2 = _mm_srai_epi16(c2, 4);
// f1 = ((b3 * 5352 + b2 * 2217 + 12000) >> 16)
// f3 = ((b3 * 2217 - b2 * 5352 + 51000) >> 16)
const __m128i b23 = _mm_unpacklo_epi16(a22, a32);
const __m128i c1 = _mm_madd_epi16(b23, k5352_2217);
const __m128i c3 = _mm_madd_epi16(b23, k2217_5352);
const __m128i d1 = _mm_add_epi32(c1, k12000_plus_one);
const __m128i d3 = _mm_add_epi32(c3, k51000);
const __m128i e1 = _mm_srai_epi32(d1, 16);
const __m128i e3 = _mm_srai_epi32(d3, 16);
const __m128i f1 = _mm_packs_epi32(e1, e1);
const __m128i f3 = _mm_packs_epi32(e3, e3);
// f1 = f1 + (a3 != 0);
// The compare will return (0xffff, 0) for (==0, !=0). To turn that into the
// desired (0, 1), we add one earlier through k12000_plus_one.
// -> f1 = f1 + 1 - (a3 == 0)
const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero));
const __m128i d0_g1 = _mm_unpacklo_epi64(d0, g1);
const __m128i d2_f3 = _mm_unpacklo_epi64(d2, f3);
_mm_storeu_si128((__m128i*)&out[0], d0_g1);
_mm_storeu_si128((__m128i*)&out[8], d2_f3);
}
}
/* Transpose bytes within elements for 64 bit elements. */
int64_t bshuf_trans_byte_elem_SSE_64(void* in, void* out, const size_t size) {
size_t ii;
char* in_b = (char*) in;
char* out_b = (char*) out;
__m128i a0, b0, c0, d0, e0, f0, g0, h0;
__m128i a1, b1, c1, d1, e1, f1, g1, h1;
for (ii=0; ii + 15 < size; ii += 16) {
a0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 0*16]);
b0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 1*16]);
c0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 2*16]);
d0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 3*16]);
e0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 4*16]);
f0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 5*16]);
g0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 6*16]);
h0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 7*16]);
a1 = _mm_unpacklo_epi8(a0, b0);
b1 = _mm_unpackhi_epi8(a0, b0);
c1 = _mm_unpacklo_epi8(c0, d0);
d1 = _mm_unpackhi_epi8(c0, d0);
e1 = _mm_unpacklo_epi8(e0, f0);
f1 = _mm_unpackhi_epi8(e0, f0);
g1 = _mm_unpacklo_epi8(g0, h0);
h1 = _mm_unpackhi_epi8(g0, h0);
a0 = _mm_unpacklo_epi8(a1, b1);
b0 = _mm_unpackhi_epi8(a1, b1);
c0 = _mm_unpacklo_epi8(c1, d1);
d0 = _mm_unpackhi_epi8(c1, d1);
e0 = _mm_unpacklo_epi8(e1, f1);
f0 = _mm_unpackhi_epi8(e1, f1);
g0 = _mm_unpacklo_epi8(g1, h1);
h0 = _mm_unpackhi_epi8(g1, h1);
a1 = _mm_unpacklo_epi32(a0, c0);
b1 = _mm_unpackhi_epi32(a0, c0);
c1 = _mm_unpacklo_epi32(b0, d0);
d1 = _mm_unpackhi_epi32(b0, d0);
e1 = _mm_unpacklo_epi32(e0, g0);
f1 = _mm_unpackhi_epi32(e0, g0);
g1 = _mm_unpacklo_epi32(f0, h0);
h1 = _mm_unpackhi_epi32(f0, h0);
a0 = _mm_unpacklo_epi64(a1, e1);
b0 = _mm_unpackhi_epi64(a1, e1);
c0 = _mm_unpacklo_epi64(b1, f1);
d0 = _mm_unpackhi_epi64(b1, f1);
e0 = _mm_unpacklo_epi64(c1, g1);
f0 = _mm_unpackhi_epi64(c1, g1);
g0 = _mm_unpacklo_epi64(d1, h1);
h0 = _mm_unpackhi_epi64(d1, h1);
_mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0);
_mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0);
_mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0);
_mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0);
_mm_storeu_si128((__m128i *) &out_b[4*size + ii], e0);
_mm_storeu_si128((__m128i *) &out_b[5*size + ii], f0);
_mm_storeu_si128((__m128i *) &out_b[6*size + ii], g0);
_mm_storeu_si128((__m128i *) &out_b[7*size + ii], h0);
}
return bshuf_trans_byte_elem_remainder(in, out, size, 8,
size - size % 16);
}
void xcorr_kernel_sse4_1(const opus_val16 * x, const opus_val16 * y, opus_val32 sum[ 4 ], int len)
{
int j;
__m128i vecX, vecX0, vecX1, vecX2, vecX3;
__m128i vecY0, vecY1, vecY2, vecY3;
__m128i sum0, sum1, sum2, sum3, vecSum;
__m128i initSum;
celt_assert(len >= 3);
sum0 = _mm_setzero_si128();
sum1 = _mm_setzero_si128();
sum2 = _mm_setzero_si128();
sum3 = _mm_setzero_si128();
for (j=0;j<(len-7);j+=8)
{
vecX = _mm_loadu_si128((__m128i *)(&x[j + 0]));
vecY0 = _mm_loadu_si128((__m128i *)(&y[j + 0]));
vecY1 = _mm_loadu_si128((__m128i *)(&y[j + 1]));
vecY2 = _mm_loadu_si128((__m128i *)(&y[j + 2]));
vecY3 = _mm_loadu_si128((__m128i *)(&y[j + 3]));
sum0 = _mm_add_epi32(sum0, _mm_madd_epi16(vecX, vecY0));
sum1 = _mm_add_epi32(sum1, _mm_madd_epi16(vecX, vecY1));
sum2 = _mm_add_epi32(sum2, _mm_madd_epi16(vecX, vecY2));
sum3 = _mm_add_epi32(sum3, _mm_madd_epi16(vecX, vecY3));
}
sum0 = _mm_add_epi32(sum0, _mm_unpackhi_epi64( sum0, sum0));
sum0 = _mm_add_epi32(sum0, _mm_shufflelo_epi16( sum0, 0x0E));
sum1 = _mm_add_epi32(sum1, _mm_unpackhi_epi64( sum1, sum1));
sum1 = _mm_add_epi32(sum1, _mm_shufflelo_epi16( sum1, 0x0E));
sum2 = _mm_add_epi32(sum2, _mm_unpackhi_epi64( sum2, sum2));
sum2 = _mm_add_epi32(sum2, _mm_shufflelo_epi16( sum2, 0x0E));
sum3 = _mm_add_epi32(sum3, _mm_unpackhi_epi64( sum3, sum3));
sum3 = _mm_add_epi32(sum3, _mm_shufflelo_epi16( sum3, 0x0E));
vecSum = _mm_unpacklo_epi64(_mm_unpacklo_epi32(sum0, sum1),
_mm_unpacklo_epi32(sum2, sum3));
for (;j<(len-3);j+=4)
{
vecX = OP_CVTEPI16_EPI32_M64(&x[j + 0]);
vecX0 = _mm_shuffle_epi32(vecX, 0x00);
vecX1 = _mm_shuffle_epi32(vecX, 0x55);
vecX2 = _mm_shuffle_epi32(vecX, 0xaa);
vecX3 = _mm_shuffle_epi32(vecX, 0xff);
vecY0 = OP_CVTEPI16_EPI32_M64(&y[j + 0]);
vecY1 = OP_CVTEPI16_EPI32_M64(&y[j + 1]);
vecY2 = OP_CVTEPI16_EPI32_M64(&y[j + 2]);
vecY3 = OP_CVTEPI16_EPI32_M64(&y[j + 3]);
sum0 = _mm_mullo_epi32(vecX0, vecY0);
sum1 = _mm_mullo_epi32(vecX1, vecY1);
sum2 = _mm_mullo_epi32(vecX2, vecY2);
sum3 = _mm_mullo_epi32(vecX3, vecY3);
sum0 = _mm_add_epi32(sum0, sum1);
sum2 = _mm_add_epi32(sum2, sum3);
vecSum = _mm_add_epi32(vecSum, sum0);
vecSum = _mm_add_epi32(vecSum, sum2);
}
for (;j<len;j++)
{
vecX = OP_CVTEPI16_EPI32_M64(&x[j + 0]);
vecX0 = _mm_shuffle_epi32(vecX, 0x00);
vecY0 = OP_CVTEPI16_EPI32_M64(&y[j + 0]);
sum0 = _mm_mullo_epi32(vecX0, vecY0);
vecSum = _mm_add_epi32(vecSum, sum0);
}
initSum = _mm_loadu_si128((__m128i *)(&sum[0]));
initSum = _mm_add_epi32(initSum, vecSum);
_mm_storeu_si128((__m128i *)sum, initSum);
}
/* SSE2 version of the rotzoom/affine warp filter */
void av1_warp_affine_sse2(int32_t *mat, uint8_t *ref, int width, int height,
int stride, uint8_t *pred, int p_col, int p_row,
int p_width, int p_height, int p_stride,
int subsampling_x, int subsampling_y, int ref_frm,
int16_t alpha, int16_t beta, int16_t gamma,
int16_t delta) {
__m128i tmp[15];
int i, j, k;
/* Note: For this code to work, the left/right frame borders need to be
extended by at least 13 pixels each. By the time we get here, other
code will have set up this border, but we allow an explicit check
for debugging purposes.
*/
/*for (i = 0; i < height; ++i) {
for (j = 0; j < 13; ++j) {
assert(ref[i * stride - 13 + j] == ref[i * stride]);
assert(ref[i * stride + width + j] == ref[i * stride + (width - 1)]);
}
}*/
for (i = 0; i < p_height; i += 8) {
for (j = 0; j < p_width; j += 8) {
// (x, y) coordinates of the center of this block in the destination
// image
int32_t dst_x = p_col + j + 4;
int32_t dst_y = p_row + i + 4;
int32_t x4, y4, ix4, sx4, iy4, sy4;
if (subsampling_x)
x4 = ROUND_POWER_OF_TWO_SIGNED(
mat[2] * 2 * dst_x + mat[3] * 2 * dst_y + mat[0] +
(mat[2] + mat[3] - (1 << WARPEDMODEL_PREC_BITS)) / 2,
1);
else
x4 = mat[2] * dst_x + mat[3] * dst_y + mat[0];
if (subsampling_y)
y4 = ROUND_POWER_OF_TWO_SIGNED(
mat[4] * 2 * dst_x + mat[5] * 2 * dst_y + mat[1] +
(mat[4] + mat[5] - (1 << WARPEDMODEL_PREC_BITS)) / 2,
1);
else
y4 = mat[4] * dst_x + mat[5] * dst_y + mat[1];
ix4 = x4 >> WARPEDMODEL_PREC_BITS;
sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
iy4 = y4 >> WARPEDMODEL_PREC_BITS;
sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
// Horizontal filter
for (k = -7; k < AOMMIN(8, p_height - i); ++k) {
int iy = iy4 + k;
if (iy < 0)
iy = 0;
else if (iy > height - 1)
iy = height - 1;
// If the block is aligned such that, after clamping, every sample
// would be taken from the leftmost/rightmost column, then we can
// skip the expensive horizontal filter.
if (ix4 <= -7) {
tmp[k + 7] = _mm_set1_epi16(
ref[iy * stride] *
(1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)));
} else if (ix4 >= width + 6) {
tmp[k + 7] = _mm_set1_epi16(
ref[iy * stride + (width - 1)] *
(1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)));
} else {
int sx = sx4 + alpha * (-4) + beta * k +
// Include rounding and offset here
(1 << (WARPEDDIFF_PREC_BITS - 1)) +
(WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
// Load source pixels
__m128i zero = _mm_setzero_si128();
__m128i src =
_mm_loadu_si128((__m128i *)(ref + iy * stride + ix4 - 7));
// Filter even-index pixels
__m128i tmp_0 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 0 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_2 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 2 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_4 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 4 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_6 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 6 * alpha) >> WARPEDDIFF_PREC_BITS)));
// coeffs 0 1 0 1 2 3 2 3 for pixels 0, 2
__m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2);
// coeffs 0 1 0 1 2 3 2 3 for pixels 4, 6
__m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6);
// coeffs 4 5 4 5 6 7 6 7 for pixels 0, 2
__m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2);
// coeffs 4 5 4 5 6 7 6 7 for pixels 4, 6
__m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6);
// coeffs 0 1 0 1 0 1 0 1 for pixels 0, 2, 4, 6
__m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10);
// coeffs 2 3 2 3 2 3 2 3 for pixels 0, 2, 4, 6
__m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10);
// coeffs 4 5 4 5 4 5 4 5 for pixels 0, 2, 4, 6
__m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14);
// coeffs 6 7 6 7 6 7 6 7 for pixels 0, 2, 4, 6
__m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14);
__m128i round_const =
_mm_set1_epi32((1 << HORSHEAR_REDUCE_PREC_BITS) >> 1);
// Calculate filtered results
__m128i src_0 = _mm_unpacklo_epi8(src, zero);
__m128i res_0 = _mm_madd_epi16(src_0, coeff_0);
__m128i src_2 = _mm_unpacklo_epi8(_mm_srli_si128(src, 2), zero);
__m128i res_2 = _mm_madd_epi16(src_2, coeff_2);
__m128i src_4 = _mm_unpacklo_epi8(_mm_srli_si128(src, 4), zero);
__m128i res_4 = _mm_madd_epi16(src_4, coeff_4);
__m128i src_6 = _mm_unpacklo_epi8(_mm_srli_si128(src, 6), zero);
__m128i res_6 = _mm_madd_epi16(src_6, coeff_6);
__m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_4),
_mm_add_epi32(res_2, res_6));
res_even = _mm_srai_epi32(_mm_add_epi32(res_even, round_const),
HORSHEAR_REDUCE_PREC_BITS);
// Filter odd-index pixels
__m128i tmp_1 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 1 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_3 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 3 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_5 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 5 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_7 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 7 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3);
__m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7);
__m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3);
__m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7);
__m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11);
__m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11);
__m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15);
__m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15);
__m128i src_1 = _mm_unpacklo_epi8(_mm_srli_si128(src, 1), zero);
__m128i res_1 = _mm_madd_epi16(src_1, coeff_1);
__m128i src_3 = _mm_unpacklo_epi8(_mm_srli_si128(src, 3), zero);
__m128i res_3 = _mm_madd_epi16(src_3, coeff_3);
__m128i src_5 = _mm_unpacklo_epi8(_mm_srli_si128(src, 5), zero);
__m128i res_5 = _mm_madd_epi16(src_5, coeff_5);
__m128i src_7 = _mm_unpacklo_epi8(_mm_srli_si128(src, 7), zero);
__m128i res_7 = _mm_madd_epi16(src_7, coeff_7);
__m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_5),
_mm_add_epi32(res_3, res_7));
res_odd = _mm_srai_epi32(_mm_add_epi32(res_odd, round_const),
HORSHEAR_REDUCE_PREC_BITS);
// Combine results into one register.
// We store the columns in the order 0, 2, 4, 6, 1, 3, 5, 7
// as this order helps with the vertical filter.
tmp[k + 7] = _mm_packs_epi32(res_even, res_odd);
}
}
// Vertical filter
for (k = -4; k < AOMMIN(4, p_height - i - 4); ++k) {
int sy = sy4 + gamma * (-4) + delta * k +
(1 << (WARPEDDIFF_PREC_BITS - 1)) +
(WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
// Load from tmp and rearrange pairs of consecutive rows into the
// column order 0 0 2 2 4 4 6 6; 1 1 3 3 5 5 7 7
__m128i *src = tmp + (k + 4);
__m128i src_0 = _mm_unpacklo_epi16(src[0], src[1]);
__m128i src_2 = _mm_unpacklo_epi16(src[2], src[3]);
__m128i src_4 = _mm_unpacklo_epi16(src[4], src[5]);
__m128i src_6 = _mm_unpacklo_epi16(src[6], src[7]);
// Filter even-index pixels
__m128i tmp_0 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 0 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_2 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 2 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_4 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 4 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_6 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 6 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2);
__m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6);
__m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2);
__m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6);
__m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10);
__m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10);
__m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14);
__m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14);
__m128i res_0 = _mm_madd_epi16(src_0, coeff_0);
__m128i res_2 = _mm_madd_epi16(src_2, coeff_2);
__m128i res_4 = _mm_madd_epi16(src_4, coeff_4);
__m128i res_6 = _mm_madd_epi16(src_6, coeff_6);
__m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_2),
_mm_add_epi32(res_4, res_6));
// Filter odd-index pixels
__m128i src_1 = _mm_unpackhi_epi16(src[0], src[1]);
__m128i src_3 = _mm_unpackhi_epi16(src[2], src[3]);
__m128i src_5 = _mm_unpackhi_epi16(src[4], src[5]);
__m128i src_7 = _mm_unpackhi_epi16(src[6], src[7]);
__m128i tmp_1 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 1 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_3 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 3 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_5 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 5 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_7 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 7 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3);
__m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7);
__m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3);
__m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7);
__m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11);
__m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11);
__m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15);
__m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15);
__m128i res_1 = _mm_madd_epi16(src_1, coeff_1);
__m128i res_3 = _mm_madd_epi16(src_3, coeff_3);
__m128i res_5 = _mm_madd_epi16(src_5, coeff_5);
__m128i res_7 = _mm_madd_epi16(src_7, coeff_7);
__m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_3),
_mm_add_epi32(res_5, res_7));
// Rearrange pixels back into the order 0 ... 7
__m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd);
__m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd);
// Round and pack into 8 bits
__m128i round_const =
_mm_set1_epi32((1 << VERSHEAR_REDUCE_PREC_BITS) >> 1);
__m128i res_lo_round = _mm_srai_epi32(
_mm_add_epi32(res_lo, round_const), VERSHEAR_REDUCE_PREC_BITS);
__m128i res_hi_round = _mm_srai_epi32(
_mm_add_epi32(res_hi, round_const), VERSHEAR_REDUCE_PREC_BITS);
__m128i res_16bit = _mm_packs_epi32(res_lo_round, res_hi_round);
__m128i res_8bit = _mm_packus_epi16(res_16bit, res_16bit);
// Store, blending with 'pred' if needed
__m128i *p = (__m128i *)&pred[(i + k + 4) * p_stride + j];
// Note: If we're outputting a 4x4 block, we need to be very careful
// to only output 4 pixels at this point, to avoid encode/decode
// mismatches when encoding with multiple threads.
if (p_width == 4) {
if (ref_frm) {
const __m128i orig = _mm_cvtsi32_si128(*(uint32_t *)p);
res_8bit = _mm_avg_epu8(res_8bit, orig);
}
*(uint32_t *)p = _mm_cvtsi128_si32(res_8bit);
} else {
if (ref_frm) res_8bit = _mm_avg_epu8(res_8bit, _mm_loadl_epi64(p));
_mm_storel_epi64(p, res_8bit);
}
}
}
}
}
/* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */
static void
unshuffle16_SSE2(uint8_t* dest, const uint8_t* orig, size_t size)
{
size_t i, j, k;
size_t neblock, numof16belem;
__m128i xmm1[16], xmm2[16];
neblock = size / 16;
numof16belem = neblock / 16;
for (i = 0, k = 0; i < numof16belem; i++, k += 16) {
/* Load the first 128 bytes in 16 XMM registrers */
for (j = 0; j < 16; j++) {
xmm1[j] = ((__m128i *)orig)[j*numof16belem+i];
}
/* Shuffle bytes */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
xmm2[j] = _mm_unpacklo_epi8(xmm1[j*2], xmm1[j*2+1]);
/* Compute the hi 32 bytes */
xmm2[8+j] = _mm_unpackhi_epi8(xmm1[j*2], xmm1[j*2+1]);
}
/* Shuffle 2-byte words */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
xmm1[j] = _mm_unpacklo_epi16(xmm2[j*2], xmm2[j*2+1]);
/* Compute the hi 32 bytes */
xmm1[8+j] = _mm_unpackhi_epi16(xmm2[j*2], xmm2[j*2+1]);
}
/* Shuffle 4-byte dwords */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
xmm2[j] = _mm_unpacklo_epi32(xmm1[j*2], xmm1[j*2+1]);
/* Compute the hi 32 bytes */
xmm2[8+j] = _mm_unpackhi_epi32(xmm1[j*2], xmm1[j*2+1]);
}
/* Shuffle 8-byte qwords */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
xmm1[j] = _mm_unpacklo_epi64(xmm2[j*2], xmm2[j*2+1]);
/* Compute the hi 32 bytes */
xmm1[8+j] = _mm_unpackhi_epi64(xmm2[j*2], xmm2[j*2+1]);
}
/* Store the result vectors in proper order */
((__m128i *)dest)[k+0] = xmm1[0];
((__m128i *)dest)[k+1] = xmm1[8];
((__m128i *)dest)[k+2] = xmm1[4];
((__m128i *)dest)[k+3] = xmm1[12];
((__m128i *)dest)[k+4] = xmm1[2];
((__m128i *)dest)[k+5] = xmm1[10];
((__m128i *)dest)[k+6] = xmm1[6];
((__m128i *)dest)[k+7] = xmm1[14];
((__m128i *)dest)[k+8] = xmm1[1];
((__m128i *)dest)[k+9] = xmm1[9];
((__m128i *)dest)[k+10] = xmm1[5];
((__m128i *)dest)[k+11] = xmm1[13];
((__m128i *)dest)[k+12] = xmm1[3];
((__m128i *)dest)[k+13] = xmm1[11];
((__m128i *)dest)[k+14] = xmm1[7];
((__m128i *)dest)[k+15] = xmm1[15];
}
}
// Hadamard transform
// Returns the weighted sum of the absolute value of transformed coefficients.
// w[] contains a row-major 4 by 4 symmetric matrix.
static int TTransform(const uint8_t* inA, const uint8_t* inB,
const uint16_t* const w) {
int32_t sum[4];
__m128i tmp_0, tmp_1, tmp_2, tmp_3;
// Load and combine inputs.
{
const __m128i inA_0 = _mm_loadu_si128((const __m128i*)&inA[BPS * 0]);
const __m128i inA_1 = _mm_loadu_si128((const __m128i*)&inA[BPS * 1]);
const __m128i inA_2 = _mm_loadu_si128((const __m128i*)&inA[BPS * 2]);
// In SSE4.1, with gcc 4.8 at least (maybe other versions),
// _mm_loadu_si128 is faster than _mm_loadl_epi64. But for the last lump
// of inA and inB, _mm_loadl_epi64 is still used not to have an out of
// bound read.
const __m128i inA_3 = _mm_loadl_epi64((const __m128i*)&inA[BPS * 3]);
const __m128i inB_0 = _mm_loadu_si128((const __m128i*)&inB[BPS * 0]);
const __m128i inB_1 = _mm_loadu_si128((const __m128i*)&inB[BPS * 1]);
const __m128i inB_2 = _mm_loadu_si128((const __m128i*)&inB[BPS * 2]);
const __m128i inB_3 = _mm_loadl_epi64((const __m128i*)&inB[BPS * 3]);
// Combine inA and inB (we'll do two transforms in parallel).
const __m128i inAB_0 = _mm_unpacklo_epi32(inA_0, inB_0);
const __m128i inAB_1 = _mm_unpacklo_epi32(inA_1, inB_1);
const __m128i inAB_2 = _mm_unpacklo_epi32(inA_2, inB_2);
const __m128i inAB_3 = _mm_unpacklo_epi32(inA_3, inB_3);
tmp_0 = _mm_cvtepu8_epi16(inAB_0);
tmp_1 = _mm_cvtepu8_epi16(inAB_1);
tmp_2 = _mm_cvtepu8_epi16(inAB_2);
tmp_3 = _mm_cvtepu8_epi16(inAB_3);
// a00 a01 a02 a03 b00 b01 b02 b03
// a10 a11 a12 a13 b10 b11 b12 b13
// a20 a21 a22 a23 b20 b21 b22 b23
// a30 a31 a32 a33 b30 b31 b32 b33
}
// Vertical pass first to avoid a transpose (vertical and horizontal passes
// are commutative because w/kWeightY is symmetric) and subsequent transpose.
{
// Calculate a and b (two 4x4 at once).
const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2);
const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3);
const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3);
const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2);
const __m128i b0 = _mm_add_epi16(a0, a1);
const __m128i b1 = _mm_add_epi16(a3, a2);
const __m128i b2 = _mm_sub_epi16(a3, a2);
const __m128i b3 = _mm_sub_epi16(a0, a1);
// a00 a01 a02 a03 b00 b01 b02 b03
// a10 a11 a12 a13 b10 b11 b12 b13
// a20 a21 a22 a23 b20 b21 b22 b23
// a30 a31 a32 a33 b30 b31 b32 b33
// Transpose the two 4x4.
VP8Transpose_2_4x4_16b(&b0, &b1, &b2, &b3, &tmp_0, &tmp_1, &tmp_2, &tmp_3);
}
// Horizontal pass and difference of weighted sums.
{
// Load all inputs.
const __m128i w_0 = _mm_loadu_si128((const __m128i*)&w[0]);
const __m128i w_8 = _mm_loadu_si128((const __m128i*)&w[8]);
// Calculate a and b (two 4x4 at once).
const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2);
const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3);
const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3);
const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2);
const __m128i b0 = _mm_add_epi16(a0, a1);
const __m128i b1 = _mm_add_epi16(a3, a2);
const __m128i b2 = _mm_sub_epi16(a3, a2);
const __m128i b3 = _mm_sub_epi16(a0, a1);
// Separate the transforms of inA and inB.
__m128i A_b0 = _mm_unpacklo_epi64(b0, b1);
__m128i A_b2 = _mm_unpacklo_epi64(b2, b3);
__m128i B_b0 = _mm_unpackhi_epi64(b0, b1);
__m128i B_b2 = _mm_unpackhi_epi64(b2, b3);
A_b0 = _mm_abs_epi16(A_b0);
A_b2 = _mm_abs_epi16(A_b2);
B_b0 = _mm_abs_epi16(B_b0);
B_b2 = _mm_abs_epi16(B_b2);
// weighted sums
A_b0 = _mm_madd_epi16(A_b0, w_0);
A_b2 = _mm_madd_epi16(A_b2, w_8);
B_b0 = _mm_madd_epi16(B_b0, w_0);
B_b2 = _mm_madd_epi16(B_b2, w_8);
A_b0 = _mm_add_epi32(A_b0, A_b2);
B_b0 = _mm_add_epi32(B_b0, B_b2);
// difference of weighted sums
A_b2 = _mm_sub_epi32(A_b0, B_b0);
_mm_storeu_si128((__m128i*)&sum[0], A_b2);
}
return sum[0] + sum[1] + sum[2] + sum[3];
}
opus_int silk_VAD_GetSA_Q8_sse4_1( /* O Return value, 0 if success */
silk_encoder_state *psEncC, /* I/O Encoder state */
const opus_int16 pIn[] /* I PCM input */
)
{
opus_int SA_Q15, pSNR_dB_Q7, input_tilt;
opus_int decimated_framelength1, decimated_framelength2;
opus_int decimated_framelength;
opus_int dec_subframe_length, dec_subframe_offset, SNR_Q7, i, b, s;
opus_int32 sumSquared, smooth_coef_Q16;
opus_int16 HPstateTmp;
VARDECL(opus_int16, X);
opus_int32 Xnrg[ VAD_N_BANDS ];
opus_int32 NrgToNoiseRatio_Q8[ VAD_N_BANDS ];
opus_int32 speech_nrg, x_tmp;
opus_int X_offset[ VAD_N_BANDS ];
opus_int ret = 0;
silk_VAD_state *psSilk_VAD = &psEncC->sVAD;
SAVE_STACK;
/* Safety checks */
silk_assert(VAD_N_BANDS == 4);
silk_assert(MAX_FRAME_LENGTH >= psEncC->frame_length);
silk_assert(psEncC->frame_length <= 512);
silk_assert(psEncC->frame_length == 8 * silk_RSHIFT(psEncC->frame_length, 3));
/***********************/
/* Filter and Decimate */
/***********************/
decimated_framelength1 = silk_RSHIFT(psEncC->frame_length, 1);
decimated_framelength2 = silk_RSHIFT(psEncC->frame_length, 2);
decimated_framelength = silk_RSHIFT(psEncC->frame_length, 3);
/* Decimate into 4 bands:
0 L 3L L 3L 5L
- -- - -- --
8 8 2 4 4
[0-1 kHz| temp. |1-2 kHz| 2-4 kHz | 4-8 kHz |
They're arranged to allow the minimal (frame_length / 4) extra
scratch space during the downsampling process */
X_offset[ 0 ] = 0;
X_offset[ 1 ] = decimated_framelength + decimated_framelength2;
X_offset[ 2 ] = X_offset[ 1 ] + decimated_framelength;
X_offset[ 3 ] = X_offset[ 2 ] + decimated_framelength2;
ALLOC(X, X_offset[ 3 ] + decimated_framelength1, opus_int16);
/* 0-8 kHz to 0-4 kHz and 4-8 kHz */
silk_ana_filt_bank_1(pIn, &psSilk_VAD->AnaState[ 0 ],
X, &X[ X_offset[ 3 ] ], psEncC->frame_length);
/* 0-4 kHz to 0-2 kHz and 2-4 kHz */
silk_ana_filt_bank_1(X, &psSilk_VAD->AnaState1[ 0 ],
X, &X[ X_offset[ 2 ] ], decimated_framelength1);
/* 0-2 kHz to 0-1 kHz and 1-2 kHz */
silk_ana_filt_bank_1(X, &psSilk_VAD->AnaState2[ 0 ],
X, &X[ X_offset[ 1 ] ], decimated_framelength2);
/*********************************************/
/* HP filter on lowest band (differentiator) */
/*********************************************/
X[ decimated_framelength - 1 ] = silk_RSHIFT(X[ decimated_framelength - 1 ], 1);
HPstateTmp = X[ decimated_framelength - 1 ];
for(i = decimated_framelength - 1; i > 0; i--) {
X[ i - 1 ] = silk_RSHIFT(X[ i - 1 ], 1);
X[ i ] -= X[ i - 1 ];
}
X[ 0 ] -= psSilk_VAD->HPstate;
psSilk_VAD->HPstate = HPstateTmp;
/*************************************/
/* Calculate the energy in each band */
/*************************************/
for(b = 0; b < VAD_N_BANDS; b++) {
/* Find the decimated framelength in the non-uniformly divided bands */
decimated_framelength = silk_RSHIFT(psEncC->frame_length, silk_min_int(VAD_N_BANDS - b, VAD_N_BANDS - 1));
/* Split length into subframe lengths */
dec_subframe_length = silk_RSHIFT(decimated_framelength, VAD_INTERNAL_SUBFRAMES_LOG2);
dec_subframe_offset = 0;
/* Compute energy per sub-frame */
/* initialize with summed energy of last subframe */
Xnrg[ b ] = psSilk_VAD->XnrgSubfr[ b ];
for(s = 0; s < VAD_INTERNAL_SUBFRAMES; s++) {
__m128i xmm_X, xmm_acc;
sumSquared = 0;
xmm_acc = _mm_setzero_si128();
for(i = 0; i < dec_subframe_length - 7; i += 8)
{
xmm_X = _mm_loadu_si128((__m128i *)&(X[ X_offset[ b ] + i + dec_subframe_offset ]));
xmm_X = _mm_srai_epi16(xmm_X, 3);
xmm_X = _mm_madd_epi16(xmm_X, xmm_X);
xmm_acc = _mm_add_epi32(xmm_acc, xmm_X);
}
xmm_acc = _mm_add_epi32(xmm_acc, _mm_unpackhi_epi64(xmm_acc, xmm_acc));
xmm_acc = _mm_add_epi32(xmm_acc, _mm_shufflelo_epi16(xmm_acc, 0x0E));
sumSquared += _mm_cvtsi128_si32(xmm_acc);
for(; i < dec_subframe_length; i++) {
/* The energy will be less than dec_subframe_length * (silk_int16_MIN / 8) ^ 2. */
/* Therefore we can accumulate with no risk of overflow (unless dec_subframe_length > 128) */
x_tmp = silk_RSHIFT(
X[ X_offset[ b ] + i + dec_subframe_offset ], 3);
sumSquared = silk_SMLABB(sumSquared, x_tmp, x_tmp);
/* Safety check */
silk_assert(sumSquared >= 0);
}
/* Add/saturate summed energy of current subframe */
if(s < VAD_INTERNAL_SUBFRAMES - 1) {
Xnrg[ b ] = silk_ADD_POS_SAT32(Xnrg[ b ], sumSquared);
} else {
/* Look-ahead subframe */
Xnrg[ b ] = silk_ADD_POS_SAT32(Xnrg[ b ], silk_RSHIFT(sumSquared, 1));
}
dec_subframe_offset += dec_subframe_length;
}
psSilk_VAD->XnrgSubfr[ b ] = sumSquared;
}
/********************/
/* Noise estimation */
/********************/
silk_VAD_GetNoiseLevels(&Xnrg[ 0 ], psSilk_VAD);
/***********************************************/
/* Signal-plus-noise to noise ratio estimation */
/***********************************************/
sumSquared = 0;
input_tilt = 0;
for(b = 0; b < VAD_N_BANDS; b++) {
speech_nrg = Xnrg[ b ] - psSilk_VAD->NL[ b ];
if(speech_nrg > 0) {
/* Divide, with sufficient resolution */
if((Xnrg[ b ] & 0xFF800000) == 0) {
NrgToNoiseRatio_Q8[ b ] = silk_DIV32(silk_LSHIFT(Xnrg[ b ], 8), psSilk_VAD->NL[ b ] + 1);
} else {
NrgToNoiseRatio_Q8[ b ] = silk_DIV32(Xnrg[ b ], silk_RSHIFT(psSilk_VAD->NL[ b ], 8) + 1);
}
/* Convert to log domain */
SNR_Q7 = silk_lin2log(NrgToNoiseRatio_Q8[ b ]) - 8 * 128;
/* Sum-of-squares */
sumSquared = silk_SMLABB(sumSquared, SNR_Q7, SNR_Q7); /* Q14 */
/* Tilt measure */
if(speech_nrg < ((opus_int32)1 << 20)) {
/* Scale down SNR value for small subband speech energies */
SNR_Q7 = silk_SMULWB(silk_LSHIFT(silk_SQRT_APPROX(speech_nrg), 6), SNR_Q7);
}
input_tilt = silk_SMLAWB(input_tilt, tiltWeights[ b ], SNR_Q7);
} else {
NrgToNoiseRatio_Q8[ b ] = 256;
}
}
/* Mean-of-squares */
sumSquared = silk_DIV32_16(sumSquared, VAD_N_BANDS); /* Q14 */
/* Root-mean-square approximation, scale to dBs, and write to output pointer */
pSNR_dB_Q7 = (opus_int16)(3 * silk_SQRT_APPROX(sumSquared)); /* Q7 */
/*********************************/
/* Speech Probability Estimation */
/*********************************/
SA_Q15 = silk_sigm_Q15(silk_SMULWB(VAD_SNR_FACTOR_Q16, pSNR_dB_Q7) - VAD_NEGATIVE_OFFSET_Q5);
/**************************/
/* Frequency Tilt Measure */
/**************************/
psEncC->input_tilt_Q15 = silk_LSHIFT(silk_sigm_Q15(input_tilt) - 16384, 1);
/**************************************************/
/* Scale the sigmoid output based on power levels */
/**************************************************/
speech_nrg = 0;
for(b = 0; b < VAD_N_BANDS; b++) {
/* Accumulate signal-without-noise energies, higher frequency bands have more weight */
speech_nrg += (b + 1) * silk_RSHIFT(Xnrg[ b ] - psSilk_VAD->NL[ b ], 4);
}
/* Power scaling */
if(speech_nrg <= 0) {
SA_Q15 = silk_RSHIFT(SA_Q15, 1);
} else if(speech_nrg < 32768) {
if(psEncC->frame_length == 10 * psEncC->fs_kHz) {
speech_nrg = silk_LSHIFT_SAT32(speech_nrg, 16);
} else {
speech_nrg = silk_LSHIFT_SAT32(speech_nrg, 15);
}
/* square-root */
speech_nrg = silk_SQRT_APPROX(speech_nrg);
SA_Q15 = silk_SMULWB(32768 + speech_nrg, SA_Q15);
}
/* Copy the resulting speech activity in Q8 */
psEncC->speech_activity_Q8 = silk_min_int(silk_RSHIFT(SA_Q15, 7), silk_uint8_MAX);
/***********************************/
/* Energy Level and SNR estimation */
/***********************************/
/* Smoothing coefficient */
smooth_coef_Q16 = silk_SMULWB(VAD_SNR_SMOOTH_COEF_Q18, silk_SMULWB((opus_int32)SA_Q15, SA_Q15));
if(psEncC->frame_length == 10 * psEncC->fs_kHz) {
smooth_coef_Q16 >>= 1;
}
void av1_highbd_jnt_convolve_2d_sse4_1(
const uint16_t *src, int src_stride, uint16_t *dst0, int dst_stride0, int w,
int h, const InterpFilterParams *filter_params_x,
const InterpFilterParams *filter_params_y, const int subpel_x_q4,
const int subpel_y_q4, ConvolveParams *conv_params, int bd) {
DECLARE_ALIGNED(16, int16_t,
im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]);
CONV_BUF_TYPE *dst = conv_params->dst;
int dst_stride = conv_params->dst_stride;
int im_h = h + filter_params_y->taps - 1;
int im_stride = MAX_SB_SIZE;
int i, j;
const int do_average = conv_params->do_average;
const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg;
const int fo_vert = filter_params_y->taps / 2 - 1;
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint16_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;
const int w0 = conv_params->fwd_offset;
const int w1 = conv_params->bck_offset;
const __m128i wt0 = _mm_set1_epi32(w0);
const __m128i wt1 = _mm_set1_epi32(w1);
const int offset_0 =
bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const int offset = (1 << offset_0) + (1 << (offset_0 - 1));
const __m128i offset_const = _mm_set1_epi32(offset);
const int rounding_shift =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const __m128i rounding_const = _mm_set1_epi32((1 << rounding_shift) >> 1);
const __m128i clip_pixel_to_bd =
_mm_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
// Check that, even with 12-bit input, the intermediate values will fit
// into an unsigned 16-bit intermediate array.
assert(bd + FILTER_BITS + 2 - conv_params->round_0 <= 16);
/* Horizontal filter */
{
const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
filter_params_x, subpel_x_q4 & SUBPEL_MASK);
const __m128i coeffs_x = _mm_loadu_si128((__m128i *)x_filter);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(coeffs_x, coeffs_x);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(coeffs_x, coeffs_x);
// coeffs 0 1 0 1 0 1 0 1
const __m128i coeff_01 = _mm_unpacklo_epi64(tmp_0, tmp_0);
// coeffs 2 3 2 3 2 3 2 3
const __m128i coeff_23 = _mm_unpackhi_epi64(tmp_0, tmp_0);
// coeffs 4 5 4 5 4 5 4 5
const __m128i coeff_45 = _mm_unpacklo_epi64(tmp_1, tmp_1);
// coeffs 6 7 6 7 6 7 6 7
const __m128i coeff_67 = _mm_unpackhi_epi64(tmp_1, tmp_1);
const __m128i round_const = _mm_set1_epi32(
((1 << conv_params->round_0) >> 1) + (1 << (bd + FILTER_BITS - 1)));
const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_0);
for (i = 0; i < im_h; ++i) {
for (j = 0; j < w; j += 8) {
const __m128i data =
_mm_loadu_si128((__m128i *)&src_ptr[i * src_stride + j]);
const __m128i data2 =
_mm_loadu_si128((__m128i *)&src_ptr[i * src_stride + j + 8]);
// Filter even-index pixels
const __m128i res_0 = _mm_madd_epi16(data, coeff_01);
const __m128i res_2 =
_mm_madd_epi16(_mm_alignr_epi8(data2, data, 4), coeff_23);
const __m128i res_4 =
_mm_madd_epi16(_mm_alignr_epi8(data2, data, 8), coeff_45);
const __m128i res_6 =
_mm_madd_epi16(_mm_alignr_epi8(data2, data, 12), coeff_67);
__m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_4),
_mm_add_epi32(res_2, res_6));
res_even =
_mm_sra_epi32(_mm_add_epi32(res_even, round_const), round_shift);
// Filter odd-index pixels
const __m128i res_1 =
_mm_madd_epi16(_mm_alignr_epi8(data2, data, 2), coeff_01);
const __m128i res_3 =
_mm_madd_epi16(_mm_alignr_epi8(data2, data, 6), coeff_23);
const __m128i res_5 =
_mm_madd_epi16(_mm_alignr_epi8(data2, data, 10), coeff_45);
const __m128i res_7 =
_mm_madd_epi16(_mm_alignr_epi8(data2, data, 14), coeff_67);
__m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_5),
_mm_add_epi32(res_3, res_7));
res_odd =
_mm_sra_epi32(_mm_add_epi32(res_odd, round_const), round_shift);
// Pack in the column order 0, 2, 4, 6, 1, 3, 5, 7
__m128i res = _mm_packs_epi32(res_even, res_odd);
_mm_storeu_si128((__m128i *)&im_block[i * im_stride + j], res);
}
}
}
/* Vertical filter */
{
const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
filter_params_y, subpel_y_q4 & SUBPEL_MASK);
const __m128i coeffs_y = _mm_loadu_si128((__m128i *)y_filter);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(coeffs_y, coeffs_y);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(coeffs_y, coeffs_y);
// coeffs 0 1 0 1 0 1 0 1
const __m128i coeff_01 = _mm_unpacklo_epi64(tmp_0, tmp_0);
// coeffs 2 3 2 3 2 3 2 3
const __m128i coeff_23 = _mm_unpackhi_epi64(tmp_0, tmp_0);
// coeffs 4 5 4 5 4 5 4 5
const __m128i coeff_45 = _mm_unpacklo_epi64(tmp_1, tmp_1);
// coeffs 6 7 6 7 6 7 6 7
const __m128i coeff_67 = _mm_unpackhi_epi64(tmp_1, tmp_1);
const __m128i round_const = _mm_set1_epi32(
((1 << conv_params->round_1) >> 1) -
(1 << (bd + 2 * FILTER_BITS - conv_params->round_0 - 1)));
const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_1);
for (i = 0; i < h; ++i) {
for (j = 0; j < w; j += 8) {
// Filter even-index pixels
const int16_t *data = &im_block[i * im_stride + j];
const __m128i src_0 =
_mm_unpacklo_epi16(*(__m128i *)(data + 0 * im_stride),
*(__m128i *)(data + 1 * im_stride));
const __m128i src_2 =
_mm_unpacklo_epi16(*(__m128i *)(data + 2 * im_stride),
*(__m128i *)(data + 3 * im_stride));
const __m128i src_4 =
_mm_unpacklo_epi16(*(__m128i *)(data + 4 * im_stride),
*(__m128i *)(data + 5 * im_stride));
const __m128i src_6 =
_mm_unpacklo_epi16(*(__m128i *)(data + 6 * im_stride),
*(__m128i *)(data + 7 * im_stride));
const __m128i res_0 = _mm_madd_epi16(src_0, coeff_01);
const __m128i res_2 = _mm_madd_epi16(src_2, coeff_23);
const __m128i res_4 = _mm_madd_epi16(src_4, coeff_45);
const __m128i res_6 = _mm_madd_epi16(src_6, coeff_67);
const __m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_2),
_mm_add_epi32(res_4, res_6));
// Filter odd-index pixels
const __m128i src_1 =
_mm_unpackhi_epi16(*(__m128i *)(data + 0 * im_stride),
*(__m128i *)(data + 1 * im_stride));
const __m128i src_3 =
_mm_unpackhi_epi16(*(__m128i *)(data + 2 * im_stride),
*(__m128i *)(data + 3 * im_stride));
const __m128i src_5 =
_mm_unpackhi_epi16(*(__m128i *)(data + 4 * im_stride),
*(__m128i *)(data + 5 * im_stride));
const __m128i src_7 =
_mm_unpackhi_epi16(*(__m128i *)(data + 6 * im_stride),
*(__m128i *)(data + 7 * im_stride));
const __m128i res_1 = _mm_madd_epi16(src_1, coeff_01);
const __m128i res_3 = _mm_madd_epi16(src_3, coeff_23);
const __m128i res_5 = _mm_madd_epi16(src_5, coeff_45);
const __m128i res_7 = _mm_madd_epi16(src_7, coeff_67);
const __m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_3),
_mm_add_epi32(res_5, res_7));
// Rearrange pixels back into the order 0 ... 7
const __m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd);
const __m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd);
const __m128i res_lo_round =
_mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift);
const __m128i res_unsigned_lo =
_mm_add_epi32(res_lo_round, offset_const);
if (w < 8) {
if (do_average) {
const __m128i data_0 =
_mm_loadl_epi64((__m128i *)(&dst[i * dst_stride + j]));
const __m128i data_ref_0 = _mm_cvtepu16_epi32(data_0);
const __m128i comp_avg_res = highbd_comp_avg_sse4_1(
&data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg);
const __m128i round_result = highbd_convolve_rounding_sse2(
&comp_avg_res, &offset_const, &rounding_const, rounding_shift);
const __m128i res_16b =
_mm_packus_epi32(round_result, round_result);
const __m128i res_clip = _mm_min_epi16(res_16b, clip_pixel_to_bd);
_mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_clip);
} else {
const __m128i res_16b =
_mm_packus_epi32(res_unsigned_lo, res_unsigned_lo);
_mm_storel_epi64((__m128i *)(&dst[i * dst_stride + j]), res_16b);
}
} else {
const __m128i res_hi_round =
_mm_sra_epi32(_mm_add_epi32(res_hi, round_const), round_shift);
const __m128i res_unsigned_hi =
_mm_add_epi32(res_hi_round, offset_const);
if (do_average) {
const __m128i data_lo =
_mm_loadl_epi64((__m128i *)(&dst[i * dst_stride + j]));
const __m128i data_hi =
_mm_loadl_epi64((__m128i *)(&dst[i * dst_stride + j + 4]));
const __m128i data_ref_0_lo = _mm_cvtepu16_epi32(data_lo);
const __m128i data_ref_0_hi = _mm_cvtepu16_epi32(data_hi);
const __m128i comp_avg_res_lo = highbd_comp_avg_sse4_1(
&data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg);
const __m128i comp_avg_res_hi = highbd_comp_avg_sse4_1(
&data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg);
const __m128i round_result_lo =
highbd_convolve_rounding_sse2(&comp_avg_res_lo, &offset_const,
&rounding_const, rounding_shift);
const __m128i round_result_hi =
highbd_convolve_rounding_sse2(&comp_avg_res_hi, &offset_const,
&rounding_const, rounding_shift);
const __m128i res_16b =
_mm_packus_epi32(round_result_lo, round_result_hi);
const __m128i res_clip = _mm_min_epi16(res_16b, clip_pixel_to_bd);
_mm_store_si128((__m128i *)(&dst0[i * dst_stride0 + j]), res_clip);
} else {
const __m128i res_16b =
_mm_packus_epi32(res_unsigned_lo, res_unsigned_hi);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_16b);
}
}
}
}
}
}
static inline void
ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq)
{
int i;
uint16_t rx_id;
volatile union ixgbe_adv_rx_desc *rxdp;
struct ixgbe_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
struct rte_mbuf *mb0, *mb1;
__m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
RTE_PKTMBUF_HEADROOM);
__m128i dma_addr0, dma_addr1;
rxdp = rxq->rx_ring + rxq->rxrearm_start;
/* Pull 'n' more MBUFs into the software ring */
if (rte_mempool_get_bulk(rxq->mb_pool,
(void *)rxep,
RTE_IXGBE_RXQ_REARM_THRESH) < 0) {
if (rxq->rxrearm_nb + RTE_IXGBE_RXQ_REARM_THRESH >=
rxq->nb_rx_desc) {
dma_addr0 = _mm_setzero_si128();
for (i = 0; i < RTE_IXGBE_DESCS_PER_LOOP; i++) {
rxep[i].mbuf = &rxq->fake_mbuf;
_mm_store_si128((__m128i *)&rxdp[i].read,
dma_addr0);
}
}
rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
RTE_IXGBE_RXQ_REARM_THRESH;
return;
}
/* Initialize the mbufs in vector, process 2 mbufs in one loop */
for (i = 0; i < RTE_IXGBE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
__m128i vaddr0, vaddr1;
uintptr_t p0, p1;
mb0 = rxep[0].mbuf;
mb1 = rxep[1].mbuf;
/*
* Flush mbuf with pkt template.
* Data to be rearmed is 6 bytes long.
* Though, RX will overwrite ol_flags that are coming next
* anyway. So overwrite whole 8 bytes with one load:
* 6 bytes of rearm_data plus first 2 bytes of ol_flags.
*/
p0 = (uintptr_t)&mb0->rearm_data;
*(uint64_t *)p0 = rxq->mbuf_initializer;
p1 = (uintptr_t)&mb1->rearm_data;
*(uint64_t *)p1 = rxq->mbuf_initializer;
/* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */
vaddr0 = _mm_loadu_si128((__m128i *)&(mb0->buf_addr));
vaddr1 = _mm_loadu_si128((__m128i *)&(mb1->buf_addr));
/* convert pa to dma_addr hdr/data */
dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
/* add headroom to pa values */
dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
/* flush desc with pa dma_addr */
_mm_store_si128((__m128i *)&rxdp++->read, dma_addr0);
_mm_store_si128((__m128i *)&rxdp++->read, dma_addr1);
}
rxq->rxrearm_start += RTE_IXGBE_RXQ_REARM_THRESH;
if (rxq->rxrearm_start >= rxq->nb_rx_desc)
rxq->rxrearm_start = 0;
rxq->rxrearm_nb -= RTE_IXGBE_RXQ_REARM_THRESH;
rx_id = (uint16_t) ((rxq->rxrearm_start == 0) ?
(rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
/* Update the tail pointer on the NIC */
IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id);
}
void c_unrolled_4x(uint8_t *keys, uint8_t *data, uint8_t *dataOut) {
__m128i mask = _mm_set_epi8(0x0C, 0x03, 0x06, 0x09, 0x08, 0x0F, 0x02, 0x05,
0x04, 0x0B, 0x0E, 0x01, 0x00, 0x07, 0x0A, 0x0D);
__m128i tmp0;
__m128i zero = _mm_xor_si128(tmp0, tmp0);
__m128i key0 = _mm_load_si128((__m128i*)&(keys[ 0]));
__m128i key1 = _mm_load_si128((__m128i*)&(keys[16]));
__m128i key2 = _mm_load_si128((__m128i*)&(keys[32]));
__m128i key3 = _mm_load_si128((__m128i*)&(keys[48]));
__m128i data0 = _mm_load_si128((__m128i*) data );
__m128i data1 = data0;
__m128i data2 = data0;
__m128i data3 = data0;
__m128i rk0;
__m128i rk1;
__m128i rk2;
__m128i rk3;
__m128i rki;
__m128i rkj;
__m128i mmrcon;
data0 = _mm_xor_si128(data0, key0);
data1 = _mm_xor_si128(data1, key1);
data2 = _mm_xor_si128(data2, key2);
data3 = _mm_xor_si128(data3, key3);
rki = _mm_unpacklo_epi32(key0, key1);
rkj = _mm_unpacklo_epi32(key2, key3);
rk0 = _mm_unpacklo_epi64(rki, rkj);
rk1 = _mm_unpackhi_epi64(rki, rkj);
rki = _mm_unpackhi_epi32(key0, key1);
rkj = _mm_unpackhi_epi32(key2, key3);
rk2 = _mm_unpacklo_epi64(rki, rkj);
rk3 = _mm_unpackhi_epi64(rki, rkj);
tmp0 = _mm_aesenclast_si128(rk3, zero);
tmp0 = _mm_shuffle_epi8(tmp0, mask);
mmrcon = _mm_set1_epi32(rcon[1]);
tmp0 = _mm_xor_si128(tmp0, mmrcon);
rk0 = _mm_xor_si128(rk0, tmp0);
rk1 = _mm_xor_si128(rk1, rk0);
rk2 = _mm_xor_si128(rk2, rk1);
rk3 = _mm_xor_si128(rk3, rk2);
rki = _mm_unpacklo_epi32(rk0, rk1);
rkj = _mm_unpacklo_epi32(rk2, rk3);
key0 = _mm_unpacklo_epi64(rki, rkj);
key1 = _mm_unpackhi_epi64(rki, rkj);
rki = _mm_unpackhi_epi32(rk0, rk1);
rkj = _mm_unpackhi_epi32(rk2, rk3);
key2 = _mm_unpacklo_epi64(rki, rkj);
key3 = _mm_unpackhi_epi64(rki, rkj);
for (uint8_t roundCounter = 1; roundCounter < 10; roundCounter++) {
tmp0 = _mm_aesenclast_si128(rk3, zero);
tmp0 = _mm_shuffle_epi8(tmp0, mask);
mmrcon = _mm_set1_epi32(rcon[roundCounter+1]);
data0 = _mm_aesenc_si128(data0, key0);
data1 = _mm_aesenc_si128(data1, key1);
data2 = _mm_aesenc_si128(data2, key2);
data3 = _mm_aesenc_si128(data3, key3);
tmp0 = _mm_xor_si128(tmp0, mmrcon);
rk0 = _mm_xor_si128(rk0, tmp0);
rk1 = _mm_xor_si128(rk1, rk0);
rk2 = _mm_xor_si128(rk2, rk1);
rk3 = _mm_xor_si128(rk3, rk2);
rki = _mm_unpacklo_epi32(rk0, rk1);
rkj = _mm_unpacklo_epi32(rk2, rk3);
key0 = _mm_unpacklo_epi64(rki, rkj);
key1 = _mm_unpackhi_epi64(rki, rkj);
rki = _mm_unpackhi_epi32(rk0, rk1);
rkj = _mm_unpackhi_epi32(rk2, rk3);
key2 = _mm_unpacklo_epi64(rki, rkj);
key3 = _mm_unpackhi_epi64(rki, rkj);
}
data0 = _mm_aesenclast_si128(data0, key0);
data1 = _mm_aesenclast_si128(data1, key1);
data2 = _mm_aesenclast_si128(data2, key2);
data3 = _mm_aesenclast_si128(data3, key3);
_mm_store_si128((__m128i*)&(dataOut[00]), data0);
_mm_store_si128((__m128i*)&(dataOut[16]), data1);
_mm_store_si128((__m128i*)&(dataOut[32]), data2);
_mm_store_si128((__m128i*)&(dataOut[48]), data3);
}
// Does one or two inverse transforms.
static void ITransform(const uint8_t* ref, const int16_t* in, uint8_t* dst,
int do_two) {
// This implementation makes use of 16-bit fixed point versions of two
// multiply constants:
// K1 = sqrt(2) * cos (pi/8) ~= 85627 / 2^16
// K2 = sqrt(2) * sin (pi/8) ~= 35468 / 2^16
//
// To be able to use signed 16-bit integers, we use the following trick to
// have constants within range:
// - Associated constants are obtained by subtracting the 16-bit fixed point
// version of one:
// k = K - (1 << 16) => K = k + (1 << 16)
// K1 = 85267 => k1 = 20091
// K2 = 35468 => k2 = -30068
// - The multiplication of a variable by a constant become the sum of the
// variable and the multiplication of that variable by the associated
// constant:
// (x * K) >> 16 = (x * (k + (1 << 16))) >> 16 = ((x * k ) >> 16) + x
const __m128i k1 = _mm_set1_epi16(20091);
const __m128i k2 = _mm_set1_epi16(-30068);
__m128i T0, T1, T2, T3;
// Load and concatenate the transform coefficients (we'll do two inverse
// transforms in parallel). In the case of only one inverse transform, the
// second half of the vectors will just contain random value we'll never
// use nor store.
__m128i in0, in1, in2, in3;
{
in0 = _mm_loadl_epi64((const __m128i*)&in[0]);
in1 = _mm_loadl_epi64((const __m128i*)&in[4]);
in2 = _mm_loadl_epi64((const __m128i*)&in[8]);
in3 = _mm_loadl_epi64((const __m128i*)&in[12]);
// a00 a10 a20 a30 x x x x
// a01 a11 a21 a31 x x x x
// a02 a12 a22 a32 x x x x
// a03 a13 a23 a33 x x x x
if (do_two) {
const __m128i inB0 = _mm_loadl_epi64((const __m128i*)&in[16]);
const __m128i inB1 = _mm_loadl_epi64((const __m128i*)&in[20]);
const __m128i inB2 = _mm_loadl_epi64((const __m128i*)&in[24]);
const __m128i inB3 = _mm_loadl_epi64((const __m128i*)&in[28]);
in0 = _mm_unpacklo_epi64(in0, inB0);
in1 = _mm_unpacklo_epi64(in1, inB1);
in2 = _mm_unpacklo_epi64(in2, inB2);
in3 = _mm_unpacklo_epi64(in3, inB3);
// a00 a10 a20 a30 b00 b10 b20 b30
// a01 a11 a21 a31 b01 b11 b21 b31
// a02 a12 a22 a32 b02 b12 b22 b32
// a03 a13 a23 a33 b03 b13 b23 b33
}
}
// Vertical pass and subsequent transpose.
{
// First pass, c and d calculations are longer because of the "trick"
// multiplications.
const __m128i a = _mm_add_epi16(in0, in2);
const __m128i b = _mm_sub_epi16(in0, in2);
// c = MUL(in1, K2) - MUL(in3, K1) = MUL(in1, k2) - MUL(in3, k1) + in1 - in3
const __m128i c1 = _mm_mulhi_epi16(in1, k2);
const __m128i c2 = _mm_mulhi_epi16(in3, k1);
const __m128i c3 = _mm_sub_epi16(in1, in3);
const __m128i c4 = _mm_sub_epi16(c1, c2);
const __m128i c = _mm_add_epi16(c3, c4);
// d = MUL(in1, K1) + MUL(in3, K2) = MUL(in1, k1) + MUL(in3, k2) + in1 + in3
const __m128i d1 = _mm_mulhi_epi16(in1, k1);
const __m128i d2 = _mm_mulhi_epi16(in3, k2);
const __m128i d3 = _mm_add_epi16(in1, in3);
const __m128i d4 = _mm_add_epi16(d1, d2);
const __m128i d = _mm_add_epi16(d3, d4);
// Second pass.
const __m128i tmp0 = _mm_add_epi16(a, d);
const __m128i tmp1 = _mm_add_epi16(b, c);
const __m128i tmp2 = _mm_sub_epi16(b, c);
const __m128i tmp3 = _mm_sub_epi16(a, d);
// Transpose the two 4x4.
// a00 a01 a02 a03 b00 b01 b02 b03
// a10 a11 a12 a13 b10 b11 b12 b13
// a20 a21 a22 a23 b20 b21 b22 b23
// a30 a31 a32 a33 b30 b31 b32 b33
const __m128i transpose0_0 = _mm_unpacklo_epi16(tmp0, tmp1);
const __m128i transpose0_1 = _mm_unpacklo_epi16(tmp2, tmp3);
const __m128i transpose0_2 = _mm_unpackhi_epi16(tmp0, tmp1);
const __m128i transpose0_3 = _mm_unpackhi_epi16(tmp2, tmp3);
// a00 a10 a01 a11 a02 a12 a03 a13
// a20 a30 a21 a31 a22 a32 a23 a33
// b00 b10 b01 b11 b02 b12 b03 b13
// b20 b30 b21 b31 b22 b32 b23 b33
const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1);
const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3);
const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3);
// a00 a10 a20 a30 a01 a11 a21 a31
// b00 b10 b20 b30 b01 b11 b21 b31
// a02 a12 a22 a32 a03 a13 a23 a33
// b02 b12 a22 b32 b03 b13 b23 b33
T0 = _mm_unpacklo_epi64(transpose1_0, transpose1_1);
T1 = _mm_unpackhi_epi64(transpose1_0, transpose1_1);
T2 = _mm_unpacklo_epi64(transpose1_2, transpose1_3);
T3 = _mm_unpackhi_epi64(transpose1_2, transpose1_3);
// a00 a10 a20 a30 b00 b10 b20 b30
// a01 a11 a21 a31 b01 b11 b21 b31
// a02 a12 a22 a32 b02 b12 b22 b32
// a03 a13 a23 a33 b03 b13 b23 b33
}
// Horizontal pass and subsequent transpose.
{
// First pass, c and d calculations are longer because of the "trick"
// multiplications.
const __m128i four = _mm_set1_epi16(4);
const __m128i dc = _mm_add_epi16(T0, four);
const __m128i a = _mm_add_epi16(dc, T2);
const __m128i b = _mm_sub_epi16(dc, T2);
// c = MUL(T1, K2) - MUL(T3, K1) = MUL(T1, k2) - MUL(T3, k1) + T1 - T3
const __m128i c1 = _mm_mulhi_epi16(T1, k2);
const __m128i c2 = _mm_mulhi_epi16(T3, k1);
const __m128i c3 = _mm_sub_epi16(T1, T3);
const __m128i c4 = _mm_sub_epi16(c1, c2);
const __m128i c = _mm_add_epi16(c3, c4);
// d = MUL(T1, K1) + MUL(T3, K2) = MUL(T1, k1) + MUL(T3, k2) + T1 + T3
const __m128i d1 = _mm_mulhi_epi16(T1, k1);
const __m128i d2 = _mm_mulhi_epi16(T3, k2);
const __m128i d3 = _mm_add_epi16(T1, T3);
const __m128i d4 = _mm_add_epi16(d1, d2);
const __m128i d = _mm_add_epi16(d3, d4);
// Second pass.
const __m128i tmp0 = _mm_add_epi16(a, d);
const __m128i tmp1 = _mm_add_epi16(b, c);
const __m128i tmp2 = _mm_sub_epi16(b, c);
const __m128i tmp3 = _mm_sub_epi16(a, d);
const __m128i shifted0 = _mm_srai_epi16(tmp0, 3);
const __m128i shifted1 = _mm_srai_epi16(tmp1, 3);
const __m128i shifted2 = _mm_srai_epi16(tmp2, 3);
const __m128i shifted3 = _mm_srai_epi16(tmp3, 3);
// Transpose the two 4x4.
// a00 a01 a02 a03 b00 b01 b02 b03
// a10 a11 a12 a13 b10 b11 b12 b13
// a20 a21 a22 a23 b20 b21 b22 b23
// a30 a31 a32 a33 b30 b31 b32 b33
const __m128i transpose0_0 = _mm_unpacklo_epi16(shifted0, shifted1);
const __m128i transpose0_1 = _mm_unpacklo_epi16(shifted2, shifted3);
const __m128i transpose0_2 = _mm_unpackhi_epi16(shifted0, shifted1);
const __m128i transpose0_3 = _mm_unpackhi_epi16(shifted2, shifted3);
// a00 a10 a01 a11 a02 a12 a03 a13
// a20 a30 a21 a31 a22 a32 a23 a33
// b00 b10 b01 b11 b02 b12 b03 b13
// b20 b30 b21 b31 b22 b32 b23 b33
const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1);
const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3);
const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3);
// a00 a10 a20 a30 a01 a11 a21 a31
// b00 b10 b20 b30 b01 b11 b21 b31
// a02 a12 a22 a32 a03 a13 a23 a33
// b02 b12 a22 b32 b03 b13 b23 b33
T0 = _mm_unpacklo_epi64(transpose1_0, transpose1_1);
T1 = _mm_unpackhi_epi64(transpose1_0, transpose1_1);
T2 = _mm_unpacklo_epi64(transpose1_2, transpose1_3);
T3 = _mm_unpackhi_epi64(transpose1_2, transpose1_3);
// a00 a10 a20 a30 b00 b10 b20 b30
// a01 a11 a21 a31 b01 b11 b21 b31
// a02 a12 a22 a32 b02 b12 b22 b32
// a03 a13 a23 a33 b03 b13 b23 b33
}
// Add inverse transform to 'ref' and store.
{
const __m128i zero = _mm_setzero_si128();
// Load the reference(s).
__m128i ref0, ref1, ref2, ref3;
if (do_two) {
// Load eight bytes/pixels per line.
ref0 = _mm_loadl_epi64((const __m128i*)&ref[0 * BPS]);
ref1 = _mm_loadl_epi64((const __m128i*)&ref[1 * BPS]);
ref2 = _mm_loadl_epi64((const __m128i*)&ref[2 * BPS]);
ref3 = _mm_loadl_epi64((const __m128i*)&ref[3 * BPS]);
} else {
// Load four bytes/pixels per line.
ref0 = _mm_cvtsi32_si128(*(const int*)&ref[0 * BPS]);
ref1 = _mm_cvtsi32_si128(*(const int*)&ref[1 * BPS]);
ref2 = _mm_cvtsi32_si128(*(const int*)&ref[2 * BPS]);
ref3 = _mm_cvtsi32_si128(*(const int*)&ref[3 * BPS]);
}
// Convert to 16b.
ref0 = _mm_unpacklo_epi8(ref0, zero);
ref1 = _mm_unpacklo_epi8(ref1, zero);
ref2 = _mm_unpacklo_epi8(ref2, zero);
ref3 = _mm_unpacklo_epi8(ref3, zero);
// Add the inverse transform(s).
ref0 = _mm_add_epi16(ref0, T0);
ref1 = _mm_add_epi16(ref1, T1);
ref2 = _mm_add_epi16(ref2, T2);
ref3 = _mm_add_epi16(ref3, T3);
// Unsigned saturate to 8b.
ref0 = _mm_packus_epi16(ref0, ref0);
ref1 = _mm_packus_epi16(ref1, ref1);
ref2 = _mm_packus_epi16(ref2, ref2);
ref3 = _mm_packus_epi16(ref3, ref3);
// Store the results.
if (do_two) {
// Store eight bytes/pixels per line.
_mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0);
_mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1);
_mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2);
_mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3);
} else {
// Store four bytes/pixels per line.
*((int32_t *)&dst[0 * BPS]) = _mm_cvtsi128_si32(ref0);
*((int32_t *)&dst[1 * BPS]) = _mm_cvtsi128_si32(ref1);
*((int32_t *)&dst[2 * BPS]) = _mm_cvtsi128_si32(ref2);
*((int32_t *)&dst[3 * BPS]) = _mm_cvtsi128_si32(ref3);
}
}
}
static void FTransformSSE2(const uint8_t* src, const uint8_t* ref,
int16_t* out) {
const __m128i zero = _mm_setzero_si128();
const __m128i seven = _mm_set1_epi16(7);
const __m128i k7500 = _mm_set1_epi32(7500);
const __m128i k14500 = _mm_set1_epi32(14500);
const __m128i k51000 = _mm_set1_epi32(51000);
const __m128i k12000_plus_one = _mm_set1_epi32(12000 + (1 << 16));
const __m128i k5352_2217 = _mm_set_epi16(5352, 2217, 5352, 2217,
5352, 2217, 5352, 2217);
const __m128i k2217_5352 = _mm_set_epi16(2217, -5352, 2217, -5352,
2217, -5352, 2217, -5352);
__m128i v01, v32;
// Difference between src and ref and initial transpose.
{
// Load src and convert to 16b.
const __m128i src0 = _mm_loadl_epi64((__m128i*)&src[0 * BPS]);
const __m128i src1 = _mm_loadl_epi64((__m128i*)&src[1 * BPS]);
const __m128i src2 = _mm_loadl_epi64((__m128i*)&src[2 * BPS]);
const __m128i src3 = _mm_loadl_epi64((__m128i*)&src[3 * BPS]);
const __m128i src_0 = _mm_unpacklo_epi8(src0, zero);
const __m128i src_1 = _mm_unpacklo_epi8(src1, zero);
const __m128i src_2 = _mm_unpacklo_epi8(src2, zero);
const __m128i src_3 = _mm_unpacklo_epi8(src3, zero);
// Load ref and convert to 16b.
const __m128i ref0 = _mm_loadl_epi64((__m128i*)&ref[0 * BPS]);
const __m128i ref1 = _mm_loadl_epi64((__m128i*)&ref[1 * BPS]);
const __m128i ref2 = _mm_loadl_epi64((__m128i*)&ref[2 * BPS]);
const __m128i ref3 = _mm_loadl_epi64((__m128i*)&ref[3 * BPS]);
const __m128i ref_0 = _mm_unpacklo_epi8(ref0, zero);
const __m128i ref_1 = _mm_unpacklo_epi8(ref1, zero);
const __m128i ref_2 = _mm_unpacklo_epi8(ref2, zero);
const __m128i ref_3 = _mm_unpacklo_epi8(ref3, zero);
// Compute difference.
const __m128i diff0 = _mm_sub_epi16(src_0, ref_0);
const __m128i diff1 = _mm_sub_epi16(src_1, ref_1);
const __m128i diff2 = _mm_sub_epi16(src_2, ref_2);
const __m128i diff3 = _mm_sub_epi16(src_3, ref_3);
// Transpose.
// 00 01 02 03 0 0 0 0
// 10 11 12 13 0 0 0 0
// 20 21 22 23 0 0 0 0
// 30 31 32 33 0 0 0 0
const __m128i transpose0_0 = _mm_unpacklo_epi16(diff0, diff1);
const __m128i transpose0_1 = _mm_unpacklo_epi16(diff2, diff3);
// 00 10 01 11 02 12 03 13
// 20 30 21 31 22 32 23 33
const __m128i v23 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
v01 = _mm_unpacklo_epi32(transpose0_0, transpose0_1);
v32 = _mm_shuffle_epi32(v23, _MM_SHUFFLE(1, 0, 3, 2));
// a02 a12 a22 a32 a03 a13 a23 a33
// a00 a10 a20 a30 a01 a11 a21 a31
// a03 a13 a23 a33 a02 a12 a22 a32
}
// First pass and subsequent transpose.
{
// Same operations are done on the (0,3) and (1,2) pairs.
// b0 = (a0 + a3) << 3
// b1 = (a1 + a2) << 3
// b3 = (a0 - a3) << 3
// b2 = (a1 - a2) << 3
const __m128i a01 = _mm_add_epi16(v01, v32);
const __m128i a32 = _mm_sub_epi16(v01, v32);
const __m128i b01 = _mm_slli_epi16(a01, 3);
const __m128i b32 = _mm_slli_epi16(a32, 3);
const __m128i b11 = _mm_unpackhi_epi64(b01, b01);
const __m128i b22 = _mm_unpackhi_epi64(b32, b32);
// e0 = b0 + b1
// e2 = b0 - b1
const __m128i e0 = _mm_add_epi16(b01, b11);
const __m128i e2 = _mm_sub_epi16(b01, b11);
const __m128i e02 = _mm_unpacklo_epi64(e0, e2);
// e1 = (b3 * 5352 + b2 * 2217 + 14500) >> 12
// e3 = (b3 * 2217 - b2 * 5352 + 7500) >> 12
const __m128i b23 = _mm_unpacklo_epi16(b22, b32);
const __m128i c1 = _mm_madd_epi16(b23, k5352_2217);
const __m128i c3 = _mm_madd_epi16(b23, k2217_5352);
const __m128i d1 = _mm_add_epi32(c1, k14500);
const __m128i d3 = _mm_add_epi32(c3, k7500);
const __m128i e1 = _mm_srai_epi32(d1, 12);
const __m128i e3 = _mm_srai_epi32(d3, 12);
const __m128i e13 = _mm_packs_epi32(e1, e3);
// Transpose.
// 00 01 02 03 20 21 22 23
// 10 11 12 13 30 31 32 33
const __m128i transpose0_0 = _mm_unpacklo_epi16(e02, e13);
const __m128i transpose0_1 = _mm_unpackhi_epi16(e02, e13);
// 00 10 01 11 02 12 03 13
// 20 30 21 31 22 32 23 33
const __m128i v23 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
v01 = _mm_unpacklo_epi32(transpose0_0, transpose0_1);
v32 = _mm_shuffle_epi32(v23, _MM_SHUFFLE(1, 0, 3, 2));
// 02 12 22 32 03 13 23 33
// 00 10 20 30 01 11 21 31
// 03 13 23 33 02 12 22 32
}
// Second pass
{
// Same operations are done on the (0,3) and (1,2) pairs.
// a0 = v0 + v3
// a1 = v1 + v2
// a3 = v0 - v3
// a2 = v1 - v2
const __m128i a01 = _mm_add_epi16(v01, v32);
const __m128i a32 = _mm_sub_epi16(v01, v32);
const __m128i a11 = _mm_unpackhi_epi64(a01, a01);
const __m128i a22 = _mm_unpackhi_epi64(a32, a32);
// d0 = (a0 + a1 + 7) >> 4;
// d2 = (a0 - a1 + 7) >> 4;
const __m128i b0 = _mm_add_epi16(a01, a11);
const __m128i b2 = _mm_sub_epi16(a01, a11);
const __m128i c0 = _mm_add_epi16(b0, seven);
const __m128i c2 = _mm_add_epi16(b2, seven);
const __m128i d0 = _mm_srai_epi16(c0, 4);
const __m128i d2 = _mm_srai_epi16(c2, 4);
// f1 = ((b3 * 5352 + b2 * 2217 + 12000) >> 16)
// f3 = ((b3 * 2217 - b2 * 5352 + 51000) >> 16)
const __m128i b23 = _mm_unpacklo_epi16(a22, a32);
const __m128i c1 = _mm_madd_epi16(b23, k5352_2217);
const __m128i c3 = _mm_madd_epi16(b23, k2217_5352);
const __m128i d1 = _mm_add_epi32(c1, k12000_plus_one);
const __m128i d3 = _mm_add_epi32(c3, k51000);
const __m128i e1 = _mm_srai_epi32(d1, 16);
const __m128i e3 = _mm_srai_epi32(d3, 16);
const __m128i f1 = _mm_packs_epi32(e1, e1);
const __m128i f3 = _mm_packs_epi32(e3, e3);
// f1 = f1 + (a3 != 0);
// The compare will return (0xffff, 0) for (==0, !=0). To turn that into the
// desired (0, 1), we add one earlier through k12000_plus_one.
const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero));
_mm_storel_epi64((__m128i*)&out[ 0], d0);
_mm_storel_epi64((__m128i*)&out[ 4], g1);
_mm_storel_epi64((__m128i*)&out[ 8], d2);
_mm_storel_epi64((__m128i*)&out[12], f3);
}
}
