/* Routine optimized for unshuffling a buffer for a type size of 2 bytes. */
static void
unshuffle2_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements)
{
static const size_t bytesoftype = 2;
size_t i;
int j;
__m256i ymm0[2], ymm1[2];
for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) {
/* Load 32 elements (64 bytes) into 2 YMM registers. */
const uint8_t* const src_for_ith_element = src + i;
for (j = 0; j < 2; j++) {
ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements)));
}
/* Shuffle bytes */
for (j = 0; j < 2; j++) {
ymm0[j] = _mm256_permute4x64_epi64(ymm0[j], 0xd8);
}
/* Compute the low 64 bytes */
ymm1[0] = _mm256_unpacklo_epi8(ymm0[0], ymm0[1]);
/* Compute the hi 64 bytes */
ymm1[1] = _mm256_unpackhi_epi8(ymm0[0], ymm0[1]);
/* Store the result vectors in proper order */
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[1]);
}
}
template <bool align> SIMD_INLINE void VectorProduct(const __m256i & vertical, const uint8_t * horizontal, uint8_t * dst)
{
__m256i _horizontal = Load<align>((__m256i*)horizontal);
__m256i lo = DivideI16By255(_mm256_mullo_epi16(vertical, _mm256_unpacklo_epi8(_horizontal, K_ZERO)));
__m256i hi = DivideI16By255(_mm256_mullo_epi16(vertical, _mm256_unpackhi_epi8(_horizontal, K_ZERO)));
Store<align>((__m256i*)dst, _mm256_packus_epi16(lo, hi));
}
static INLINE unsigned int masked_sad32xh_avx2(
const uint8_t *src_ptr, int src_stride, const uint8_t *a_ptr, int a_stride,
const uint8_t *b_ptr, int b_stride, const uint8_t *m_ptr, int m_stride,
int width, int height) {
int x, y;
__m256i res = _mm256_setzero_si256();
const __m256i mask_max = _mm256_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
const __m256i round_scale =
_mm256_set1_epi16(1 << (15 - AOM_BLEND_A64_ROUND_BITS));
for (y = 0; y < height; y++) {
for (x = 0; x < width; x += 32) {
const __m256i src = _mm256_lddqu_si256((const __m256i *)&src_ptr[x]);
const __m256i a = _mm256_lddqu_si256((const __m256i *)&a_ptr[x]);
const __m256i b = _mm256_lddqu_si256((const __m256i *)&b_ptr[x]);
const __m256i m = _mm256_lddqu_si256((const __m256i *)&m_ptr[x]);
const __m256i m_inv = _mm256_sub_epi8(mask_max, m);
// Calculate 16 predicted pixels.
// Note that the maximum value of any entry of 'pred_l' or 'pred_r'
// is 64 * 255, so we have plenty of space to add rounding constants.
const __m256i data_l = _mm256_unpacklo_epi8(a, b);
const __m256i mask_l = _mm256_unpacklo_epi8(m, m_inv);
__m256i pred_l = _mm256_maddubs_epi16(data_l, mask_l);
pred_l = _mm256_mulhrs_epi16(pred_l, round_scale);
const __m256i data_r = _mm256_unpackhi_epi8(a, b);
const __m256i mask_r = _mm256_unpackhi_epi8(m, m_inv);
__m256i pred_r = _mm256_maddubs_epi16(data_r, mask_r);
pred_r = _mm256_mulhrs_epi16(pred_r, round_scale);
const __m256i pred = _mm256_packus_epi16(pred_l, pred_r);
res = _mm256_add_epi32(res, _mm256_sad_epu8(pred, src));
}
src_ptr += src_stride;
a_ptr += a_stride;
b_ptr += b_stride;
m_ptr += m_stride;
}
// At this point, we have two 32-bit partial SADs in lanes 0 and 2 of 'res'.
res = _mm256_shuffle_epi32(res, 0xd8);
res = _mm256_permute4x64_epi64(res, 0xd8);
res = _mm256_hadd_epi32(res, res);
res = _mm256_hadd_epi32(res, res);
int32_t sad = _mm256_extract_epi32(res, 0);
return (sad + 31) >> 6;
}
/* Routine optimized for shuffling a buffer for a type size of 16 bytes. */
static void
shuffle16_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements)
{
static const size_t bytesoftype = 16;
size_t j;
int k, l;
__m256i ymm0[16], ymm1[16];
/* Create the shuffle mask.
NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from
most to least significant (i.e., their order is reversed when compared to
loading the mask from an array). */
const __m256i shmask = _mm256_set_epi8(
0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04,
0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00,
0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04,
0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00);
for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) {
/* Fetch 32 elements (512 bytes) into 16 YMM registers. */
for (k = 0; k < 16; k++) {
ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i))));
}
/* Transpose bytes */
for (k = 0, l = 0; k < 8; k++, l +=2) {
ymm1[k*2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l+1]);
ymm1[k*2+1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l+1]);
}
/* Transpose words */
for (k = 0, l = -2; k < 8; k++, l++) {
if ((k%2) == 0) l += 2;
ymm0[k*2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l+2]);
ymm0[k*2+1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l+2]);
}
/* Transpose double words */
for (k = 0, l = -4; k < 8; k++, l++) {
if ((k%4) == 0) l += 4;
ymm1[k*2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l+4]);
ymm1[k*2+1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l+4]);
}
/* Transpose quad words */
for (k = 0; k < 8; k++) {
ymm0[k*2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k+8]);
ymm0[k*2+1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k+8]);
}
for (k = 0; k < 16; k++) {
ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8);
ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask);
}
/* Store the result vectors */
uint8_t* const dest_for_jth_element = dest + j;
for (k = 0; k < 16; k++) {
_mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm0[k]);
}
}
}
/* Routine optimized for unshuffling a buffer for a type size of 8 bytes. */
static void
unshuffle8_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements)
{
static const size_t bytesoftype = 8;
size_t i;
int j;
__m256i ymm0[8], ymm1[8];
for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) {
/* Fetch 32 elements (256 bytes) into 8 YMM registers. */
const uint8_t* const src_for_ith_element = src + i;
for (j = 0; j < 8; j++) {
ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements)));
}
/* Shuffle bytes */
for (j = 0; j < 4; j++) {
/* Compute the low 32 bytes */
ymm1[j] = _mm256_unpacklo_epi8(ymm0[j*2], ymm0[j*2+1]);
/* Compute the hi 32 bytes */
ymm1[4+j] = _mm256_unpackhi_epi8(ymm0[j*2], ymm0[j*2+1]);
}
/* Shuffle words */
for (j = 0; j < 4; j++) {
/* Compute the low 32 bytes */
ymm0[j] = _mm256_unpacklo_epi16(ymm1[j*2], ymm1[j*2+1]);
/* Compute the hi 32 bytes */
ymm0[4+j] = _mm256_unpackhi_epi16(ymm1[j*2], ymm1[j*2+1]);
}
for (j = 0; j < 8; j++) {
ymm0[j] = _mm256_permute4x64_epi64(ymm0[j], 0xd8);
}
/* Shuffle 4-byte dwords */
for (j = 0; j < 4; j++) {
/* Compute the low 32 bytes */
ymm1[j] = _mm256_unpacklo_epi32(ymm0[j*2], ymm0[j*2+1]);
/* Compute the hi 32 bytes */
ymm1[4+j] = _mm256_unpackhi_epi32(ymm0[j*2], ymm0[j*2+1]);
}
/* Store the result vectors in proper order */
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[2]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (2 * sizeof(__m256i))), ymm1[1]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (3 * sizeof(__m256i))), ymm1[3]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (4 * sizeof(__m256i))), ymm1[4]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (5 * sizeof(__m256i))), ymm1[6]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (6 * sizeof(__m256i))), ymm1[5]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (7 * sizeof(__m256i))), ymm1[7]);
}
}
static INLINE void comp_mask_pred_line_avx2(const __m256i s0, const __m256i s1,
const __m256i a,
uint8_t *comp_pred) {
const __m256i alpha_max = _mm256_set1_epi8(AOM_BLEND_A64_MAX_ALPHA);
const int16_t round_bits = 15 - AOM_BLEND_A64_ROUND_BITS;
const __m256i round_offset = _mm256_set1_epi16(1 << (round_bits));
const __m256i ma = _mm256_sub_epi8(alpha_max, a);
const __m256i ssAL = _mm256_unpacklo_epi8(s0, s1);
const __m256i aaAL = _mm256_unpacklo_epi8(a, ma);
const __m256i ssAH = _mm256_unpackhi_epi8(s0, s1);
const __m256i aaAH = _mm256_unpackhi_epi8(a, ma);
const __m256i blendAL = _mm256_maddubs_epi16(ssAL, aaAL);
const __m256i blendAH = _mm256_maddubs_epi16(ssAH, aaAH);
const __m256i roundAL = _mm256_mulhrs_epi16(blendAL, round_offset);
const __m256i roundAH = _mm256_mulhrs_epi16(blendAH, round_offset);
const __m256i roundA = _mm256_packus_epi16(roundAL, roundAH);
_mm256_storeu_si256((__m256i *)(comp_pred), roundA);
}
/* Routine optimized for shuffling a buffer for a type size of 4 bytes. */
static void
shuffle4_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements)
{
static const size_t bytesoftype = 4;
size_t i;
int j;
__m256i ymm0[4], ymm1[4];
/* Create the shuffle mask.
NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from
most to least significant (i.e., their order is reversed when compared to
loading the mask from an array). */
const __m256i mask = _mm256_set_epi32(
0x07, 0x03, 0x06, 0x02, 0x05, 0x01, 0x04, 0x00);
for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) {
/* Fetch 32 elements (128 bytes) then transpose bytes and words. */
for (j = 0; j < 4; j++) {
ymm0[j] = _mm256_loadu_si256((__m256i*)(src + (i * bytesoftype) + (j * sizeof(__m256i))));
ymm1[j] = _mm256_shuffle_epi32(ymm0[j], 0xd8);
ymm0[j] = _mm256_shuffle_epi32(ymm0[j], 0x8d);
ymm0[j] = _mm256_unpacklo_epi8(ymm1[j], ymm0[j]);
ymm1[j] = _mm256_shuffle_epi32(ymm0[j], 0x04e);
ymm0[j] = _mm256_unpacklo_epi16(ymm0[j], ymm1[j]);
}
/* Transpose double words */
for (j = 0; j < 2; j++) {
ymm1[j*2] = _mm256_unpacklo_epi32(ymm0[j*2], ymm0[j*2+1]);
ymm1[j*2+1] = _mm256_unpackhi_epi32(ymm0[j*2], ymm0[j*2+1]);
}
/* Transpose quad words */
for (j = 0; j < 2; j++) {
ymm0[j*2] = _mm256_unpacklo_epi64(ymm1[j], ymm1[j+2]);
ymm0[j*2+1] = _mm256_unpackhi_epi64(ymm1[j], ymm1[j+2]);
}
for (j = 0; j < 4; j++) {
ymm0[j] = _mm256_permutevar8x32_epi32(ymm0[j], mask);
}
/* Store the result vectors */
uint8_t* const dest_for_ith_element = dest + i;
for (j = 0; j < 4; j++) {
_mm256_storeu_si256((__m256i*)(dest_for_ith_element + (j * total_elements)), ymm0[j]);
}
}
}
/* Routine optimized for shuffling a buffer for a type size of 8 bytes. */
static void
shuffle8_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements)
{
static const size_t bytesoftype = 8;
size_t j;
int k, l;
__m256i ymm0[8], ymm1[8];
for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) {
/* Fetch 32 elements (256 bytes) then transpose bytes. */
for (k = 0; k < 8; k++) {
ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i))));
ymm1[k] = _mm256_shuffle_epi32(ymm0[k], 0x4e);
ymm1[k] = _mm256_unpacklo_epi8(ymm0[k], ymm1[k]);
}
/* Transpose words */
for (k = 0, l = 0; k < 4; k++, l +=2) {
ymm0[k*2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l+1]);
ymm0[k*2+1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l+1]);
}
/* Transpose double words */
for (k = 0, l = 0; k < 4; k++, l++) {
if (k == 2) l += 2;
ymm1[k*2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l+2]);
ymm1[k*2+1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l+2]);
}
/* Transpose quad words */
for (k = 0; k < 4; k++) {
ymm0[k*2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k+4]);
ymm0[k*2+1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k+4]);
}
for(k = 0; k < 8; k++) {
ymm1[k] = _mm256_permute4x64_epi64(ymm0[k], 0x72);
ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xD8);
ymm0[k] = _mm256_unpacklo_epi16(ymm0[k], ymm1[k]);
}
/* Store the result vectors */
uint8_t* const dest_for_jth_element = dest + j;
for (k = 0; k < 8; k++) {
_mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm0[k]);
}
}
}
static INLINE void variance_kernel_avx2(const __m256i src, const __m256i ref,
__m256i *const sse,
__m256i *const sum) {
const __m256i adj_sub = _mm256_set1_epi16(0xff01); // (1,-1)
// unpack into pairs of source and reference values
const __m256i src_ref0 = _mm256_unpacklo_epi8(src, ref);
const __m256i src_ref1 = _mm256_unpackhi_epi8(src, ref);
// subtract adjacent elements using src*1 + ref*-1
const __m256i diff0 = _mm256_maddubs_epi16(src_ref0, adj_sub);
const __m256i diff1 = _mm256_maddubs_epi16(src_ref1, adj_sub);
const __m256i madd0 = _mm256_madd_epi16(diff0, diff0);
const __m256i madd1 = _mm256_madd_epi16(diff1, diff1);
// add to the running totals
*sum = _mm256_add_epi16(*sum, _mm256_add_epi16(diff0, diff1));
*sse = _mm256_add_epi32(*sse, _mm256_add_epi32(madd0, madd1));
}
/* Routine optimized for unshuffling a buffer for a type size of 4 bytes. */
static void
unshuffle4_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements)
{
static const size_t bytesoftype = 4;
size_t i;
int j;
__m256i ymm0[4], ymm1[4];
for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) {
/* Load 32 elements (128 bytes) into 4 YMM registers. */
const uint8_t* const src_for_ith_element = src + i;
for (j = 0; j < 4; j++) {
ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements)));
}
/* Shuffle bytes */
for (j = 0; j < 2; j++) {
/* Compute the low 64 bytes */
ymm1[j] = _mm256_unpacklo_epi8(ymm0[j*2], ymm0[j*2+1]);
/* Compute the hi 64 bytes */
ymm1[2+j] = _mm256_unpackhi_epi8(ymm0[j*2], ymm0[j*2+1]);
}
/* Shuffle 2-byte words */
for (j = 0; j < 2; j++) {
/* Compute the low 64 bytes */
ymm0[j] = _mm256_unpacklo_epi16(ymm1[j*2], ymm1[j*2+1]);
/* Compute the hi 64 bytes */
ymm0[2+j] = _mm256_unpackhi_epi16(ymm1[j*2], ymm1[j*2+1]);
}
ymm1[0] = _mm256_permute2x128_si256(ymm0[0], ymm0[2], 0x20);
ymm1[1] = _mm256_permute2x128_si256(ymm0[1], ymm0[3], 0x20);
ymm1[2] = _mm256_permute2x128_si256(ymm0[0], ymm0[2], 0x31);
ymm1[3] = _mm256_permute2x128_si256(ymm0[1], ymm0[3], 0x31);
/* Store the result vectors in proper order */
for (j = 0; j < 4; j++) {
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (j * sizeof(__m256i))), ymm1[j]);
}
}
}
/* For data organized into a row for each bit (8 * elem_size rows), transpose
* the bytes. */
int64_t bshuf_trans_byte_bitrow_AVX(void* in, void* out, const size_t size,
const size_t elem_size) {
size_t hh, ii, jj, kk, mm;
char* in_b = (char*) in;
char* out_b = (char*) out;
CHECK_MULT_EIGHT(size);
size_t nrows = 8 * elem_size;
size_t nbyte_row = size / 8;
if (elem_size % 4) return bshuf_trans_byte_bitrow_SSE(in, out, size,
elem_size);
__m256i ymm_0[8];
__m256i ymm_1[8];
__m256i ymm_storeage[8][4];
for (jj = 0; jj + 31 < nbyte_row; jj += 32) {
for (ii = 0; ii + 3 < elem_size; ii += 4) {
for (hh = 0; hh < 4; hh ++) {
for (kk = 0; kk < 8; kk ++){
ymm_0[kk] = _mm256_loadu_si256((__m256i *) &in_b[
(ii * 8 + hh * 8 + kk) * nbyte_row + jj]);
}
for (kk = 0; kk < 4; kk ++){
ymm_1[kk] = _mm256_unpacklo_epi8(ymm_0[kk * 2],
ymm_0[kk * 2 + 1]);
ymm_1[kk + 4] = _mm256_unpackhi_epi8(ymm_0[kk * 2],
ymm_0[kk * 2 + 1]);
}
for (kk = 0; kk < 2; kk ++){
for (mm = 0; mm < 2; mm ++){
ymm_0[kk * 4 + mm] = _mm256_unpacklo_epi16(
ymm_1[kk * 4 + mm * 2],
ymm_1[kk * 4 + mm * 2 + 1]);
ymm_0[kk * 4 + mm + 2] = _mm256_unpackhi_epi16(
ymm_1[kk * 4 + mm * 2],
ymm_1[kk * 4 + mm * 2 + 1]);
}
}
for (kk = 0; kk < 4; kk ++){
ymm_1[kk * 2] = _mm256_unpacklo_epi32(ymm_0[kk * 2],
ymm_0[kk * 2 + 1]);
ymm_1[kk * 2 + 1] = _mm256_unpackhi_epi32(ymm_0[kk * 2],
ymm_0[kk * 2 + 1]);
}
for (kk = 0; kk < 8; kk ++){
ymm_storeage[kk][hh] = ymm_1[kk];
}
}
for (mm = 0; mm < 8; mm ++) {
for (kk = 0; kk < 4; kk ++){
ymm_0[kk] = ymm_storeage[mm][kk];
}
ymm_1[0] = _mm256_unpacklo_epi64(ymm_0[0], ymm_0[1]);
ymm_1[1] = _mm256_unpacklo_epi64(ymm_0[2], ymm_0[3]);
ymm_1[2] = _mm256_unpackhi_epi64(ymm_0[0], ymm_0[1]);
ymm_1[3] = _mm256_unpackhi_epi64(ymm_0[2], ymm_0[3]);
ymm_0[0] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 32);
ymm_0[1] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 32);
ymm_0[2] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 49);
ymm_0[3] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 49);
_mm256_storeu_si256((__m256i *) &out_b[
(jj + mm * 2 + 0 * 16) * nrows + ii * 8], ymm_0[0]);
_mm256_storeu_si256((__m256i *) &out_b[
(jj + mm * 2 + 0 * 16 + 1) * nrows + ii * 8], ymm_0[1]);
_mm256_storeu_si256((__m256i *) &out_b[
(jj + mm * 2 + 1 * 16) * nrows + ii * 8], ymm_0[2]);
_mm256_storeu_si256((__m256i *) &out_b[
(jj + mm * 2 + 1 * 16 + 1) * nrows + ii * 8], ymm_0[3]);
}
}
}
for (ii = 0; ii < nrows; ii ++ ) {
for (jj = nbyte_row - nbyte_row % 32; jj < nbyte_row; jj ++) {
out_b[jj * nrows + ii] = in_b[ii * nbyte_row + jj];
}
}
return size * elem_size;
}
void aom_highbd_comp_mask_pred_avx2(uint8_t *comp_pred8, const uint8_t *pred8,
int width, int height, const uint8_t *ref8,
int ref_stride, const uint8_t *mask,
int mask_stride, int invert_mask) {
int i = 0;
uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
const uint16_t *src0 = invert_mask ? pred : ref;
const uint16_t *src1 = invert_mask ? ref : pred;
const int stride0 = invert_mask ? width : ref_stride;
const int stride1 = invert_mask ? ref_stride : width;
const __m256i zero = _mm256_setzero_si256();
if (width == 8) {
do {
const __m256i s0 = mm256_loadu2_16(src0 + stride0, src0);
const __m256i s1 = mm256_loadu2_16(src1 + stride1, src1);
const __m128i m_l = _mm_loadl_epi64((const __m128i *)mask);
const __m128i m_h = _mm_loadl_epi64((const __m128i *)(mask + 8));
__m256i m = _mm256_castsi128_si256(m_l);
m = _mm256_insertf128_si256(m, m_h, 1);
const __m256i m_16 = _mm256_unpacklo_epi8(m, zero);
const __m256i comp = highbd_comp_mask_pred_line_avx2(s0, s1, m_16);
_mm_storeu_si128((__m128i *)(comp_pred), _mm256_castsi256_si128(comp));
_mm_storeu_si128((__m128i *)(comp_pred + width),
_mm256_extractf128_si256(comp, 1));
src0 += (stride0 << 1);
src1 += (stride1 << 1);
mask += (mask_stride << 1);
comp_pred += (width << 1);
i += 2;
} while (i < height);
} else if (width == 16) {
do {
const __m256i s0 = _mm256_loadu_si256((const __m256i *)(src0));
const __m256i s1 = _mm256_loadu_si256((const __m256i *)(src1));
const __m256i m_16 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)mask));
const __m256i comp = highbd_comp_mask_pred_line_avx2(s0, s1, m_16);
_mm256_storeu_si256((__m256i *)comp_pred, comp);
src0 += stride0;
src1 += stride1;
mask += mask_stride;
comp_pred += width;
i += 1;
} while (i < height);
} else if (width == 32) {
do {
const __m256i s0 = _mm256_loadu_si256((const __m256i *)src0);
const __m256i s2 = _mm256_loadu_si256((const __m256i *)(src0 + 16));
const __m256i s1 = _mm256_loadu_si256((const __m256i *)src1);
const __m256i s3 = _mm256_loadu_si256((const __m256i *)(src1 + 16));
const __m256i m01_16 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)mask));
const __m256i m23_16 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)(mask + 16)));
const __m256i comp = highbd_comp_mask_pred_line_avx2(s0, s1, m01_16);
const __m256i comp1 = highbd_comp_mask_pred_line_avx2(s2, s3, m23_16);
_mm256_storeu_si256((__m256i *)comp_pred, comp);
_mm256_storeu_si256((__m256i *)(comp_pred + 16), comp1);
src0 += stride0;
src1 += stride1;
mask += mask_stride;
comp_pred += width;
i += 1;
} while (i < height);
}
}
static void vpx_filter_block1d16_v8_avx2(const uint8_t *src_ptr,
ptrdiff_t src_pitch,
uint8_t *output_ptr,
ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m256i addFilterReg64;
__m256i srcReg32b1, srcReg32b2, srcReg32b3, srcReg32b4, srcReg32b5;
__m256i srcReg32b6, srcReg32b7, srcReg32b8, srcReg32b9, srcReg32b10;
__m256i srcReg32b11, srcReg32b12, filtersReg32;
__m256i firstFilters, secondFilters, thirdFilters, forthFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
// create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
addFilterReg64 = _mm256_set1_epi32((int)0x0400040u);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg =_mm_packs_epi16(filtersReg, filtersReg);
// have the same data in both lanes of a 256 bit register
filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
// duplicate only the first 16 bits (first and second byte)
// across 256 bit register
firstFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x100u));
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x504u));
// duplicate only the forth 16 bits (seventh and eighth byte)
// across 256 bit register
forthFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x706u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
// load 16 bytes 7 times in stride of src_pitch
srcReg32b1 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr)));
srcReg32b2 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch)));
srcReg32b3 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)));
srcReg32b4 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)));
srcReg32b5 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)));
srcReg32b6 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)));
srcReg32b7 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)));
// have each consecutive loads on the same 256 register
srcReg32b1 = _mm256_inserti128_si256(srcReg32b1,
_mm256_castsi256_si128(srcReg32b2), 1);
srcReg32b2 = _mm256_inserti128_si256(srcReg32b2,
_mm256_castsi256_si128(srcReg32b3), 1);
srcReg32b3 = _mm256_inserti128_si256(srcReg32b3,
_mm256_castsi256_si128(srcReg32b4), 1);
srcReg32b4 = _mm256_inserti128_si256(srcReg32b4,
_mm256_castsi256_si128(srcReg32b5), 1);
srcReg32b5 = _mm256_inserti128_si256(srcReg32b5,
_mm256_castsi256_si128(srcReg32b6), 1);
srcReg32b6 = _mm256_inserti128_si256(srcReg32b6,
_mm256_castsi256_si128(srcReg32b7), 1);
// merge every two consecutive registers except the last one
srcReg32b10 = _mm256_unpacklo_epi8(srcReg32b1, srcReg32b2);
srcReg32b1 = _mm256_unpackhi_epi8(srcReg32b1, srcReg32b2);
// save
srcReg32b11 = _mm256_unpacklo_epi8(srcReg32b3, srcReg32b4);
// save
srcReg32b3 = _mm256_unpackhi_epi8(srcReg32b3, srcReg32b4);
// save
srcReg32b2 = _mm256_unpacklo_epi8(srcReg32b5, srcReg32b6);
// save
srcReg32b5 = _mm256_unpackhi_epi8(srcReg32b5, srcReg32b6);
for (i = output_height; i > 1; i-=2) {
// load the last 2 loads of 16 bytes and have every two
// consecutive loads in the same 256 bit register
srcReg32b8 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7)));
srcReg32b7 = _mm256_inserti128_si256(srcReg32b7,
_mm256_castsi256_si128(srcReg32b8), 1);
srcReg32b9 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 8)));
srcReg32b8 = _mm256_inserti128_si256(srcReg32b8,
_mm256_castsi256_si128(srcReg32b9), 1);
// merge every two consecutive registers
// save
srcReg32b4 = _mm256_unpacklo_epi8(srcReg32b7, srcReg32b8);
srcReg32b7 = _mm256_unpackhi_epi8(srcReg32b7, srcReg32b8);
// multiply 2 adjacent elements with the filter and add the result
srcReg32b10 = _mm256_maddubs_epi16(srcReg32b10, firstFilters);
srcReg32b6 = _mm256_maddubs_epi16(srcReg32b4, forthFilters);
// add and saturate the results together
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, srcReg32b6);
// multiply 2 adjacent elements with the filter and add the result
srcReg32b8 = _mm256_maddubs_epi16(srcReg32b11, secondFilters);
srcReg32b12 = _mm256_maddubs_epi16(srcReg32b2, thirdFilters);
// add and saturate the results together
srcReg32b10 = _mm256_adds_epi16(srcReg32b10,
_mm256_min_epi16(srcReg32b8, srcReg32b12));
srcReg32b10 = _mm256_adds_epi16(srcReg32b10,
_mm256_max_epi16(srcReg32b8, srcReg32b12));
// multiply 2 adjacent elements with the filter and add the result
srcReg32b1 = _mm256_maddubs_epi16(srcReg32b1, firstFilters);
srcReg32b6 = _mm256_maddubs_epi16(srcReg32b7, forthFilters);
srcReg32b1 = _mm256_adds_epi16(srcReg32b1, srcReg32b6);
// multiply 2 adjacent elements with the filter and add the result
srcReg32b8 = _mm256_maddubs_epi16(srcReg32b3, secondFilters);
srcReg32b12 = _mm256_maddubs_epi16(srcReg32b5, thirdFilters);
// add and saturate the results together
srcReg32b1 = _mm256_adds_epi16(srcReg32b1,
_mm256_min_epi16(srcReg32b8, srcReg32b12));
srcReg32b1 = _mm256_adds_epi16(srcReg32b1,
_mm256_max_epi16(srcReg32b8, srcReg32b12));
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, addFilterReg64);
srcReg32b1 = _mm256_adds_epi16(srcReg32b1, addFilterReg64);
// shift by 7 bit each 16 bit
srcReg32b10 = _mm256_srai_epi16(srcReg32b10, 7);
srcReg32b1 = _mm256_srai_epi16(srcReg32b1, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcReg32b1 = _mm256_packus_epi16(srcReg32b10, srcReg32b1);
src_ptr+=src_stride;
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr,
_mm256_castsi256_si128(srcReg32b1));
// save the next 16 bits
_mm_store_si128((__m128i*)(output_ptr+out_pitch),
_mm256_extractf128_si256(srcReg32b1, 1));
output_ptr+=dst_stride;
// save part of the registers for next strides
srcReg32b10 = srcReg32b11;
srcReg32b1 = srcReg32b3;
srcReg32b11 = srcReg32b2;
srcReg32b3 = srcReg32b5;
srcReg32b2 = srcReg32b4;
srcReg32b5 = srcReg32b7;
srcReg32b7 = srcReg32b9;
}
if (i > 0) {
__m128i srcRegFilt1, srcRegFilt3, srcRegFilt4, srcRegFilt5;
__m128i srcRegFilt6, srcRegFilt7, srcRegFilt8;
// load the last 16 bytes
srcRegFilt8 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7));
// merge the last 2 results together
srcRegFilt4 = _mm_unpacklo_epi8(
_mm256_castsi256_si128(srcReg32b7), srcRegFilt8);
srcRegFilt7 = _mm_unpackhi_epi8(
_mm256_castsi256_si128(srcReg32b7), srcRegFilt8);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt1 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b10),
_mm256_castsi256_si128(firstFilters));
srcRegFilt4 = _mm_maddubs_epi16(srcRegFilt4,
_mm256_castsi256_si128(forthFilters));
srcRegFilt3 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b1),
_mm256_castsi256_si128(firstFilters));
srcRegFilt7 = _mm_maddubs_epi16(srcRegFilt7,
_mm256_castsi256_si128(forthFilters));
// add and saturate the results together
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt4);
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3, srcRegFilt7);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt4 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b11),
_mm256_castsi256_si128(secondFilters));
srcRegFilt5 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b3),
_mm256_castsi256_si128(secondFilters));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt6 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b2),
_mm256_castsi256_si128(thirdFilters));
srcRegFilt7 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b5),
_mm256_castsi256_si128(thirdFilters));
// add and saturate the results together
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1,
_mm_min_epi16(srcRegFilt4, srcRegFilt6));
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3,
_mm_min_epi16(srcRegFilt5, srcRegFilt7));
// add and saturate the results together
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1,
_mm_max_epi16(srcRegFilt4, srcRegFilt6));
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3,
_mm_max_epi16(srcRegFilt5, srcRegFilt7));
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1,
_mm256_castsi256_si128(addFilterReg64));
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3,
_mm256_castsi256_si128(addFilterReg64));
// shift by 7 bit each 16 bit
srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7);
srcRegFilt3 = _mm_srai_epi16(srcRegFilt3, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt3);
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr, srcRegFilt1);
}
}
/* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */
static void
unshuffle16_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements)
{
static const size_t bytesoftype = 16;
size_t i;
int j;
__m256i ymm0[16], ymm1[16];
for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) {
/* Fetch 32 elements (512 bytes) into 16 YMM registers. */
const uint8_t* const src_for_ith_element = src + i;
for (j = 0; j < 16; j++) {
ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements)));
}
/* Shuffle bytes */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
ymm1[j] = _mm256_unpacklo_epi8(ymm0[j*2], ymm0[j*2+1]);
/* Compute the hi 32 bytes */
ymm1[8+j] = _mm256_unpackhi_epi8(ymm0[j*2], ymm0[j*2+1]);
}
/* Shuffle 2-byte words */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
ymm0[j] = _mm256_unpacklo_epi16(ymm1[j*2], ymm1[j*2+1]);
/* Compute the hi 32 bytes */
ymm0[8+j] = _mm256_unpackhi_epi16(ymm1[j*2], ymm1[j*2+1]);
}
/* Shuffle 4-byte dwords */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
ymm1[j] = _mm256_unpacklo_epi32(ymm0[j*2], ymm0[j*2+1]);
/* Compute the hi 32 bytes */
ymm1[8+j] = _mm256_unpackhi_epi32(ymm0[j*2], ymm0[j*2+1]);
}
/* Shuffle 8-byte qwords */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
ymm0[j] = _mm256_unpacklo_epi64(ymm1[j*2], ymm1[j*2+1]);
/* Compute the hi 32 bytes */
ymm0[8+j] = _mm256_unpackhi_epi64(ymm1[j*2], ymm1[j*2+1]);
}
for (j = 0; j < 8; j++) {
ymm1[j] = _mm256_permute2x128_si256(ymm0[j], ymm0[j+8], 0x20);
ymm1[j+8] = _mm256_permute2x128_si256(ymm0[j], ymm0[j+8], 0x31);
}
/* Store the result vectors in proper order */
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[4]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (2 * sizeof(__m256i))), ymm1[2]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (3 * sizeof(__m256i))), ymm1[6]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (4 * sizeof(__m256i))), ymm1[1]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (5 * sizeof(__m256i))), ymm1[5]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (6 * sizeof(__m256i))), ymm1[3]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (7 * sizeof(__m256i))), ymm1[7]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (8 * sizeof(__m256i))), ymm1[8]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (9 * sizeof(__m256i))), ymm1[12]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (10 * sizeof(__m256i))), ymm1[10]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (11 * sizeof(__m256i))), ymm1[14]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (12 * sizeof(__m256i))), ymm1[9]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (13 * sizeof(__m256i))), ymm1[13]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (14 * sizeof(__m256i))), ymm1[11]);
_mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (15 * sizeof(__m256i))), ymm1[15]);
}
}
/* Routine optimized for shuffling a buffer for a type size larger than 16 bytes. */
static void
shuffle16_tiled_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements, const size_t bytesoftype)
{
size_t j;
int k, l;
__m256i ymm0[16], ymm1[16];
const lldiv_t vecs_per_el = lldiv(bytesoftype, sizeof(__m128i));
/* Create the shuffle mask.
NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from
most to least significant (i.e., their order is reversed when compared to
loading the mask from an array). */
const __m256i shmask = _mm256_set_epi8(
0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04,
0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00,
0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04,
0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00);
for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) {
/* Advance the offset into the type by the vector size (in bytes), unless this is
the initial iteration and the type size is not a multiple of the vector size.
In that case, only advance by the number of bytes necessary so that the number
of remaining bytes in the type will be a multiple of the vector size. */
size_t offset_into_type;
for (offset_into_type = 0; offset_into_type < bytesoftype;
offset_into_type += (offset_into_type == 0 && vecs_per_el.rem > 0 ? vecs_per_el.rem : sizeof(__m128i))) {
/* Fetch elements in groups of 512 bytes */
const uint8_t* const src_with_offset = src + offset_into_type;
for (k = 0; k < 16; k++) {
ymm0[k] = _mm256_loadu2_m128i(
(__m128i*)(src_with_offset + (j + (2 * k) + 1) * bytesoftype),
(__m128i*)(src_with_offset + (j + (2 * k)) * bytesoftype));
}
/* Transpose bytes */
for (k = 0, l = 0; k < 8; k++, l +=2) {
ymm1[k*2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l+1]);
ymm1[k*2+1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l+1]);
}
/* Transpose words */
for (k = 0, l = -2; k < 8; k++, l++) {
if ((k%2) == 0) l += 2;
ymm0[k*2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l+2]);
ymm0[k*2+1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l+2]);
}
/* Transpose double words */
for (k = 0, l = -4; k < 8; k++, l++) {
if ((k%4) == 0) l += 4;
ymm1[k*2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l+4]);
ymm1[k*2+1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l+4]);
}
/* Transpose quad words */
for (k = 0; k < 8; k++) {
ymm0[k*2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k+8]);
ymm0[k*2+1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k+8]);
}
for (k = 0; k < 16; k++) {
ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8);
ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask);
}
/* Store the result vectors */
uint8_t* const dest_for_jth_element = dest + j;
for (k = 0; k < 16; k++) {
_mm256_storeu_si256((__m256i*)(dest_for_jth_element + (total_elements * (offset_into_type + k))), ymm0[k]);
}
}
}
}
static inline __m256i _mm256_unpacklo_epi8_rpl(__m256i a, __m256i b) {
__m256i an = _mm256_permute4x64_epi64(a, _MM_SHUFFLE(1,1,0,0));
__m256i bn = _mm256_permute4x64_epi64(b, _MM_SHUFFLE(1,1,0,0));
return _mm256_unpacklo_epi8(an, bn);
}
/* Routine optimized for unshuffling a buffer for a type size larger than 16 bytes. */
static void
unshuffle16_tiled_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements, const size_t bytesoftype)
{
size_t i;
int j;
__m256i ymm0[16], ymm1[16];
const lldiv_t vecs_per_el = lldiv(bytesoftype, sizeof(__m128i));
/* The unshuffle loops are inverted (compared to shuffle_tiled16_avx2)
to optimize cache utilization. */
size_t offset_into_type;
for (offset_into_type = 0; offset_into_type < bytesoftype;
offset_into_type += (offset_into_type == 0 && vecs_per_el.rem > 0 ? vecs_per_el.rem : sizeof(__m128i))) {
for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) {
/* Load the first 16 bytes of 32 adjacent elements (512 bytes) into 16 YMM registers */
const uint8_t* const src_for_ith_element = src + i;
for (j = 0; j < 16; j++) {
ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (total_elements * (offset_into_type + j))));
}
/* Shuffle bytes */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
ymm1[j] = _mm256_unpacklo_epi8(ymm0[j*2], ymm0[j*2+1]);
/* Compute the hi 32 bytes */
ymm1[8+j] = _mm256_unpackhi_epi8(ymm0[j*2], ymm0[j*2+1]);
}
/* Shuffle 2-byte words */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
ymm0[j] = _mm256_unpacklo_epi16(ymm1[j*2], ymm1[j*2+1]);
/* Compute the hi 32 bytes */
ymm0[8+j] = _mm256_unpackhi_epi16(ymm1[j*2], ymm1[j*2+1]);
}
/* Shuffle 4-byte dwords */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
ymm1[j] = _mm256_unpacklo_epi32(ymm0[j*2], ymm0[j*2+1]);
/* Compute the hi 32 bytes */
ymm1[8+j] = _mm256_unpackhi_epi32(ymm0[j*2], ymm0[j*2+1]);
}
/* Shuffle 8-byte qwords */
for (j = 0; j < 8; j++) {
/* Compute the low 32 bytes */
ymm0[j] = _mm256_unpacklo_epi64(ymm1[j*2], ymm1[j*2+1]);
/* Compute the hi 32 bytes */
ymm0[8+j] = _mm256_unpackhi_epi64(ymm1[j*2], ymm1[j*2+1]);
}
for (j = 0; j < 8; j++) {
ymm1[j] = _mm256_permute2x128_si256(ymm0[j], ymm0[j+8], 0x20);
ymm1[j+8] = _mm256_permute2x128_si256(ymm0[j], ymm0[j+8], 0x31);
}
/* Store the result vectors in proper order */
const uint8_t* const dest_with_offset = dest + offset_into_type;
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x01) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x00) * bytesoftype), ymm1[0]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x03) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x02) * bytesoftype), ymm1[4]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x05) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x04) * bytesoftype), ymm1[2]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x07) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x06) * bytesoftype), ymm1[6]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x09) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x08) * bytesoftype), ymm1[1]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x0b) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x0a) * bytesoftype), ymm1[5]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x0d) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x0c) * bytesoftype), ymm1[3]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x0f) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x0e) * bytesoftype), ymm1[7]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x11) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x10) * bytesoftype), ymm1[8]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x13) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x12) * bytesoftype), ymm1[12]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x15) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x14) * bytesoftype), ymm1[10]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x17) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x16) * bytesoftype), ymm1[14]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x19) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x18) * bytesoftype), ymm1[9]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x1b) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x1a) * bytesoftype), ymm1[13]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x1d) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x1c) * bytesoftype), ymm1[11]);
_mm256_storeu2_m128i(
(__m128i*)(dest_with_offset + (i + 0x1f) * bytesoftype),
(__m128i*)(dest_with_offset + (i + 0x1e) * bytesoftype), ymm1[15]);
}
}
}
SIMD_INLINE __m256i Average16(const __m256i & s0, const __m256i & s1)
{
return _mm256_srli_epi16(_mm256_add_epi16(_mm256_add_epi16(
_mm256_hadd_epi16(_mm256_unpacklo_epi8(s0, K_ZERO), _mm256_unpackhi_epi8(s0, K_ZERO)),
_mm256_hadd_epi16(_mm256_unpacklo_epi8(s1, K_ZERO), _mm256_unpackhi_epi8(s1, K_ZERO))), K16_0002), 2);
}
__m256i test_mm256_unpacklo_epi8(__m256i a, __m256i b) {
// CHECK: shufflevector <32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
return _mm256_unpacklo_epi8(a, b);
}
template<bool align> SIMD_INLINE void StoreUnpacked(__m256i value, uint8_t * dst)
{
Store<align>((__m256i*)(dst + 0), _mm256_unpacklo_epi8(value, value));
Store<align>((__m256i*)(dst + A), _mm256_unpackhi_epi8(value, value));
}
