static INLINE void variance16_kernel_avx2(
const uint8_t *const src, const int src_stride, const uint8_t *const ref,
const int ref_stride, __m256i *const sse, __m256i *const sum) {
const __m128i s0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride));
const __m128i s1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride));
const __m128i r0 = _mm_loadu_si128((__m128i const *)(ref + 0 * ref_stride));
const __m128i r1 = _mm_loadu_si128((__m128i const *)(ref + 1 * ref_stride));
const __m256i s = _mm256_inserti128_si256(_mm256_castsi128_si256(s0), s1, 1);
const __m256i r = _mm256_inserti128_si256(_mm256_castsi128_si256(r0), r1, 1);
variance_kernel_avx2(s, r, sse, sum);
}
static void satd_8bit_4x4_dual_avx2(
const pred_buffer preds, const kvz_pixel * const orig, unsigned num_modes, unsigned *satds_out)
{
__m256i original = _mm256_broadcastsi128_si256(_mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)orig)));
__m256i pred = _mm256_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)preds[0]));
pred = _mm256_inserti128_si256(pred, _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)preds[1])), 1);
__m256i diff_lo = _mm256_sub_epi16(pred, original);
original = _mm256_broadcastsi128_si256(_mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)(orig + 8))));
pred = _mm256_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)(preds[0] + 8)));
pred = _mm256_inserti128_si256(pred, _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)(preds[1] + 8))), 1);
__m256i diff_hi = _mm256_sub_epi16(pred, original);
//Hor
__m256i row0 = _mm256_hadd_epi16(diff_lo, diff_hi);
__m256i row1 = _mm256_hsub_epi16(diff_lo, diff_hi);
__m256i row2 = _mm256_hadd_epi16(row0, row1);
__m256i row3 = _mm256_hsub_epi16(row0, row1);
//Ver
row0 = _mm256_hadd_epi16(row2, row3);
row1 = _mm256_hsub_epi16(row2, row3);
row2 = _mm256_hadd_epi16(row0, row1);
row3 = _mm256_hsub_epi16(row0, row1);
//Abs and sum
row2 = _mm256_abs_epi16(row2);
row3 = _mm256_abs_epi16(row3);
row3 = _mm256_add_epi16(row2, row3);
row3 = _mm256_add_epi16(row3, _mm256_shuffle_epi32(row3, KVZ_PERMUTE(2, 3, 0, 1) ));
row3 = _mm256_add_epi16(row3, _mm256_shuffle_epi32(row3, KVZ_PERMUTE(1, 0, 1, 0) ));
row3 = _mm256_add_epi16(row3, _mm256_shufflelo_epi16(row3, KVZ_PERMUTE(1, 0, 1, 0) ));
unsigned sum1 = _mm_extract_epi16(_mm256_castsi256_si128(row3), 0);
sum1 = (sum1 + 1) >> 1;
unsigned sum2 = _mm_extract_epi16(_mm256_extracti128_si256(row3, 1), 0);
sum2 = (sum2 + 1) >> 1;
satds_out[0] = sum1;
satds_out[1] = sum2;
}
static void
sfid_render_cache_rt_write_simd8_bgra_unorm8_xmajor(struct thread *t,
const struct sfid_render_cache_args *args)
{
__m256i argb;
const float scale = 255.0f;
struct reg src[4];
memcpy(src, &t->grf[args->src], sizeof(src));
const int cpp = 4;
const int slice_y = args->rt.minimum_array_element * args->rt.qpitch;
const int x = t->grf[1].uw[4];
const int y = t->grf[1].uw[5] + slice_y;
void *base = xmajor_offset(args->rt.pixels, x, y, args->rt.stride, cpp);
if (gt.blend.enable) {
/* Load unorm8 */
__m128i lo = _mm_load_si128(base);
__m128i hi = _mm_load_si128(base + 512);
__m256i dst_argb = _mm256_inserti128_si256(_mm256_castsi128_si256(lo), hi, 1);
dst_argb = _mm256_permute4x64_epi64(dst_argb, SWIZZLE(0, 2, 1, 3));
blend_unorm8_argb(src, dst_argb);
}
gamma_correct(args->rt.format, src);
const __m256i r = to_unorm(src[0].reg, scale);
const __m256i g = to_unorm(src[1].reg, scale);
const __m256i b = to_unorm(src[2].reg, scale);
const __m256i a = to_unorm(src[3].reg, scale);
argb = _mm256_slli_epi32(a, 8);
argb = _mm256_or_si256(argb, r);
argb = _mm256_slli_epi32(argb, 8);
argb = _mm256_or_si256(argb, g);
argb = _mm256_slli_epi32(argb, 8);
argb = _mm256_or_si256(argb, b);
/* Swizzle two middle pixel pairs so that dword 0-3 and 4-7
* form linear owords of pixels. */
argb = _mm256_permute4x64_epi64(argb, SWIZZLE(0, 2, 1, 3));
__m256i mask = _mm256_permute4x64_epi64(t->mask_q1, SWIZZLE(0, 2, 1, 3));
_mm_maskstore_epi32(base,
_mm256_extractf128_si256(mask, 0),
_mm256_extractf128_si256(argb, 0));
_mm_maskstore_epi32(base + 512,
_mm256_extractf128_si256(mask, 1),
_mm256_extractf128_si256(argb, 1));
}
__m256i test_mm256_inserti128_si256(__m256i a, __m128i b) {
// CHECK: @llvm.x86.avx2.vinserti128
return _mm256_inserti128_si256(a, b, 1);
}
// Immediate should be truncated to one bit.
__m256i test2_mm256_inserti128_si256(__m256i a, __m128i b) {
// CHECK-LABEL: test2_mm256_inserti128_si256
// CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
// CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
return _mm256_inserti128_si256(a, b, 2);
}
static void vpx_filter_block1d16_h8_avx2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr,
ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m256i addFilterReg64, filt1Reg, filt2Reg, filt3Reg, filt4Reg;
__m256i firstFilters, secondFilters, thirdFilters, forthFilters;
__m256i srcRegFilt32b1_1, srcRegFilt32b2_1, srcRegFilt32b2, srcRegFilt32b3;
__m256i srcReg32b1, srcReg32b2, filtersReg32;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
// create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
addFilterReg64 = _mm256_set1_epi32((int)0x0400040u);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg =_mm_packs_epi16(filtersReg, filtersReg);
// have the same data in both lanes of a 256 bit register
filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
// duplicate only the first 16 bits (first and second byte)
// across 256 bit register
firstFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x100u));
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x504u));
// duplicate only the forth 16 bits (seventh and eighth byte)
// across 256 bit register
forthFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x706u));
filt1Reg = _mm256_load_si256((__m256i const *)filt1_global_avx2);
filt2Reg = _mm256_load_si256((__m256i const *)filt2_global_avx2);
filt3Reg = _mm256_load_si256((__m256i const *)filt3_global_avx2);
filt4Reg = _mm256_load_si256((__m256i const *)filt4_global_avx2);
// multiple the size of the source and destination stride by two
src_stride = src_pixels_per_line << 1;
dst_stride = output_pitch << 1;
for (i = output_height; i > 1; i-=2) {
// load the 2 strides of source
srcReg32b1 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr - 3)));
srcReg32b1 = _mm256_inserti128_si256(srcReg32b1,
_mm_loadu_si128((const __m128i *)
(src_ptr+src_pixels_per_line-3)), 1);
// filter the source buffer
srcRegFilt32b1_1= _mm256_shuffle_epi8(srcReg32b1, filt1Reg);
srcRegFilt32b2= _mm256_shuffle_epi8(srcReg32b1, filt4Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b1_1 = _mm256_maddubs_epi16(srcRegFilt32b1_1, firstFilters);
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, forthFilters);
// add and saturate the results together
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, srcRegFilt32b2);
// filter the source buffer
srcRegFilt32b3= _mm256_shuffle_epi8(srcReg32b1, filt2Reg);
srcRegFilt32b2= _mm256_shuffle_epi8(srcReg32b1, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters);
// add and saturate the results together
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1,
_mm256_min_epi16(srcRegFilt32b3, srcRegFilt32b2));
// reading 2 strides of the next 16 bytes
// (part of it was being read by earlier read)
srcReg32b2 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + 5)));
srcReg32b2 = _mm256_inserti128_si256(srcReg32b2,
_mm_loadu_si128((const __m128i *)
(src_ptr+src_pixels_per_line+5)), 1);
// add and saturate the results together
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1,
_mm256_max_epi16(srcRegFilt32b3, srcRegFilt32b2));
// filter the source buffer
srcRegFilt32b2_1 = _mm256_shuffle_epi8(srcReg32b2, filt1Reg);
srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b2, filt4Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b2_1 = _mm256_maddubs_epi16(srcRegFilt32b2_1, firstFilters);
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, forthFilters);
// add and saturate the results together
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, srcRegFilt32b2);
// filter the source buffer
srcRegFilt32b3= _mm256_shuffle_epi8(srcReg32b2, filt2Reg);
srcRegFilt32b2= _mm256_shuffle_epi8(srcReg32b2, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters);
// add and saturate the results together
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1,
_mm256_min_epi16(srcRegFilt32b3, srcRegFilt32b2));
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1,
_mm256_max_epi16(srcRegFilt32b3, srcRegFilt32b2));
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, addFilterReg64);
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, addFilterReg64);
// shift by 7 bit each 16 bit
srcRegFilt32b1_1 = _mm256_srai_epi16(srcRegFilt32b1_1, 7);
srcRegFilt32b2_1 = _mm256_srai_epi16(srcRegFilt32b2_1, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcRegFilt32b1_1 = _mm256_packus_epi16(srcRegFilt32b1_1,
srcRegFilt32b2_1);
src_ptr+=src_stride;
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr,
_mm256_castsi256_si128(srcRegFilt32b1_1));
// save the next 16 bits
_mm_store_si128((__m128i*)(output_ptr+output_pitch),
_mm256_extractf128_si256(srcRegFilt32b1_1, 1));
output_ptr+=dst_stride;
}
// if the number of strides is odd.
// process only 16 bytes
if (i > 0) {
__m128i srcReg1, srcReg2, srcRegFilt1_1, srcRegFilt2_1;
__m128i srcRegFilt2, srcRegFilt3;
srcReg1 = _mm_loadu_si128((const __m128i *)(src_ptr - 3));
// filter the source buffer
srcRegFilt1_1 = _mm_shuffle_epi8(srcReg1,
_mm256_castsi256_si128(filt1Reg));
srcRegFilt2 = _mm_shuffle_epi8(srcReg1,
_mm256_castsi256_si128(filt4Reg));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt1_1 = _mm_maddubs_epi16(srcRegFilt1_1,
_mm256_castsi256_si128(firstFilters));
srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2,
_mm256_castsi256_si128(forthFilters));
// add and saturate the results together
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1, srcRegFilt2);
// filter the source buffer
srcRegFilt3= _mm_shuffle_epi8(srcReg1,
_mm256_castsi256_si128(filt2Reg));
srcRegFilt2= _mm_shuffle_epi8(srcReg1,
_mm256_castsi256_si128(filt3Reg));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt3 = _mm_maddubs_epi16(srcRegFilt3,
_mm256_castsi256_si128(secondFilters));
srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2,
_mm256_castsi256_si128(thirdFilters));
// add and saturate the results together
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1,
_mm_min_epi16(srcRegFilt3, srcRegFilt2));
// reading the next 16 bytes
// (part of it was being read by earlier read)
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + 5));
// add and saturate the results together
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1,
_mm_max_epi16(srcRegFilt3, srcRegFilt2));
// filter the source buffer
srcRegFilt2_1 = _mm_shuffle_epi8(srcReg2,
_mm256_castsi256_si128(filt1Reg));
srcRegFilt2 = _mm_shuffle_epi8(srcReg2,
_mm256_castsi256_si128(filt4Reg));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt2_1 = _mm_maddubs_epi16(srcRegFilt2_1,
_mm256_castsi256_si128(firstFilters));
srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2,
_mm256_castsi256_si128(forthFilters));
// add and saturate the results together
srcRegFilt2_1 = _mm_adds_epi16(srcRegFilt2_1, srcRegFilt2);
// filter the source buffer
srcRegFilt3 = _mm_shuffle_epi8(srcReg2,
_mm256_castsi256_si128(filt2Reg));
srcRegFilt2 = _mm_shuffle_epi8(srcReg2,
_mm256_castsi256_si128(filt3Reg));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt3 = _mm_maddubs_epi16(srcRegFilt3,
_mm256_castsi256_si128(secondFilters));
srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2,
_mm256_castsi256_si128(thirdFilters));
// add and saturate the results together
srcRegFilt2_1 = _mm_adds_epi16(srcRegFilt2_1,
_mm_min_epi16(srcRegFilt3, srcRegFilt2));
srcRegFilt2_1 = _mm_adds_epi16(srcRegFilt2_1,
_mm_max_epi16(srcRegFilt3, srcRegFilt2));
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1,
_mm256_castsi256_si128(addFilterReg64));
srcRegFilt2_1 = _mm_adds_epi16(srcRegFilt2_1,
_mm256_castsi256_si128(addFilterReg64));
// shift by 7 bit each 16 bit
srcRegFilt1_1 = _mm_srai_epi16(srcRegFilt1_1, 7);
srcRegFilt2_1 = _mm_srai_epi16(srcRegFilt2_1, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcRegFilt1_1 = _mm_packus_epi16(srcRegFilt1_1, srcRegFilt2_1);
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr, srcRegFilt1_1);
}
}
static void vpx_filter_block1d16_v8_avx2(const uint8_t *src_ptr,
ptrdiff_t src_pitch,
uint8_t *output_ptr,
ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m256i addFilterReg64;
__m256i srcReg32b1, srcReg32b2, srcReg32b3, srcReg32b4, srcReg32b5;
__m256i srcReg32b6, srcReg32b7, srcReg32b8, srcReg32b9, srcReg32b10;
__m256i srcReg32b11, srcReg32b12, filtersReg32;
__m256i firstFilters, secondFilters, thirdFilters, forthFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
// create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
addFilterReg64 = _mm256_set1_epi32((int)0x0400040u);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg =_mm_packs_epi16(filtersReg, filtersReg);
// have the same data in both lanes of a 256 bit register
filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
// duplicate only the first 16 bits (first and second byte)
// across 256 bit register
firstFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x100u));
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x504u));
// duplicate only the forth 16 bits (seventh and eighth byte)
// across 256 bit register
forthFilters = _mm256_shuffle_epi8(filtersReg32,
_mm256_set1_epi16(0x706u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
// load 16 bytes 7 times in stride of src_pitch
srcReg32b1 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr)));
srcReg32b2 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch)));
srcReg32b3 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)));
srcReg32b4 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)));
srcReg32b5 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)));
srcReg32b6 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)));
srcReg32b7 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)));
// have each consecutive loads on the same 256 register
srcReg32b1 = _mm256_inserti128_si256(srcReg32b1,
_mm256_castsi256_si128(srcReg32b2), 1);
srcReg32b2 = _mm256_inserti128_si256(srcReg32b2,
_mm256_castsi256_si128(srcReg32b3), 1);
srcReg32b3 = _mm256_inserti128_si256(srcReg32b3,
_mm256_castsi256_si128(srcReg32b4), 1);
srcReg32b4 = _mm256_inserti128_si256(srcReg32b4,
_mm256_castsi256_si128(srcReg32b5), 1);
srcReg32b5 = _mm256_inserti128_si256(srcReg32b5,
_mm256_castsi256_si128(srcReg32b6), 1);
srcReg32b6 = _mm256_inserti128_si256(srcReg32b6,
_mm256_castsi256_si128(srcReg32b7), 1);
// merge every two consecutive registers except the last one
srcReg32b10 = _mm256_unpacklo_epi8(srcReg32b1, srcReg32b2);
srcReg32b1 = _mm256_unpackhi_epi8(srcReg32b1, srcReg32b2);
// save
srcReg32b11 = _mm256_unpacklo_epi8(srcReg32b3, srcReg32b4);
// save
srcReg32b3 = _mm256_unpackhi_epi8(srcReg32b3, srcReg32b4);
// save
srcReg32b2 = _mm256_unpacklo_epi8(srcReg32b5, srcReg32b6);
// save
srcReg32b5 = _mm256_unpackhi_epi8(srcReg32b5, srcReg32b6);
for (i = output_height; i > 1; i-=2) {
// load the last 2 loads of 16 bytes and have every two
// consecutive loads in the same 256 bit register
srcReg32b8 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7)));
srcReg32b7 = _mm256_inserti128_si256(srcReg32b7,
_mm256_castsi256_si128(srcReg32b8), 1);
srcReg32b9 = _mm256_castsi128_si256(
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 8)));
srcReg32b8 = _mm256_inserti128_si256(srcReg32b8,
_mm256_castsi256_si128(srcReg32b9), 1);
// merge every two consecutive registers
// save
srcReg32b4 = _mm256_unpacklo_epi8(srcReg32b7, srcReg32b8);
srcReg32b7 = _mm256_unpackhi_epi8(srcReg32b7, srcReg32b8);
// multiply 2 adjacent elements with the filter and add the result
srcReg32b10 = _mm256_maddubs_epi16(srcReg32b10, firstFilters);
srcReg32b6 = _mm256_maddubs_epi16(srcReg32b4, forthFilters);
// add and saturate the results together
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, srcReg32b6);
// multiply 2 adjacent elements with the filter and add the result
srcReg32b8 = _mm256_maddubs_epi16(srcReg32b11, secondFilters);
srcReg32b12 = _mm256_maddubs_epi16(srcReg32b2, thirdFilters);
// add and saturate the results together
srcReg32b10 = _mm256_adds_epi16(srcReg32b10,
_mm256_min_epi16(srcReg32b8, srcReg32b12));
srcReg32b10 = _mm256_adds_epi16(srcReg32b10,
_mm256_max_epi16(srcReg32b8, srcReg32b12));
// multiply 2 adjacent elements with the filter and add the result
srcReg32b1 = _mm256_maddubs_epi16(srcReg32b1, firstFilters);
srcReg32b6 = _mm256_maddubs_epi16(srcReg32b7, forthFilters);
srcReg32b1 = _mm256_adds_epi16(srcReg32b1, srcReg32b6);
// multiply 2 adjacent elements with the filter and add the result
srcReg32b8 = _mm256_maddubs_epi16(srcReg32b3, secondFilters);
srcReg32b12 = _mm256_maddubs_epi16(srcReg32b5, thirdFilters);
// add and saturate the results together
srcReg32b1 = _mm256_adds_epi16(srcReg32b1,
_mm256_min_epi16(srcReg32b8, srcReg32b12));
srcReg32b1 = _mm256_adds_epi16(srcReg32b1,
_mm256_max_epi16(srcReg32b8, srcReg32b12));
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, addFilterReg64);
srcReg32b1 = _mm256_adds_epi16(srcReg32b1, addFilterReg64);
// shift by 7 bit each 16 bit
srcReg32b10 = _mm256_srai_epi16(srcReg32b10, 7);
srcReg32b1 = _mm256_srai_epi16(srcReg32b1, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcReg32b1 = _mm256_packus_epi16(srcReg32b10, srcReg32b1);
src_ptr+=src_stride;
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr,
_mm256_castsi256_si128(srcReg32b1));
// save the next 16 bits
_mm_store_si128((__m128i*)(output_ptr+out_pitch),
_mm256_extractf128_si256(srcReg32b1, 1));
output_ptr+=dst_stride;
// save part of the registers for next strides
srcReg32b10 = srcReg32b11;
srcReg32b1 = srcReg32b3;
srcReg32b11 = srcReg32b2;
srcReg32b3 = srcReg32b5;
srcReg32b2 = srcReg32b4;
srcReg32b5 = srcReg32b7;
srcReg32b7 = srcReg32b9;
}
if (i > 0) {
__m128i srcRegFilt1, srcRegFilt3, srcRegFilt4, srcRegFilt5;
__m128i srcRegFilt6, srcRegFilt7, srcRegFilt8;
// load the last 16 bytes
srcRegFilt8 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 7));
// merge the last 2 results together
srcRegFilt4 = _mm_unpacklo_epi8(
_mm256_castsi256_si128(srcReg32b7), srcRegFilt8);
srcRegFilt7 = _mm_unpackhi_epi8(
_mm256_castsi256_si128(srcReg32b7), srcRegFilt8);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt1 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b10),
_mm256_castsi256_si128(firstFilters));
srcRegFilt4 = _mm_maddubs_epi16(srcRegFilt4,
_mm256_castsi256_si128(forthFilters));
srcRegFilt3 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b1),
_mm256_castsi256_si128(firstFilters));
srcRegFilt7 = _mm_maddubs_epi16(srcRegFilt7,
_mm256_castsi256_si128(forthFilters));
// add and saturate the results together
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt4);
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3, srcRegFilt7);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt4 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b11),
_mm256_castsi256_si128(secondFilters));
srcRegFilt5 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b3),
_mm256_castsi256_si128(secondFilters));
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt6 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b2),
_mm256_castsi256_si128(thirdFilters));
srcRegFilt7 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b5),
_mm256_castsi256_si128(thirdFilters));
// add and saturate the results together
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1,
_mm_min_epi16(srcRegFilt4, srcRegFilt6));
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3,
_mm_min_epi16(srcRegFilt5, srcRegFilt7));
// add and saturate the results together
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1,
_mm_max_epi16(srcRegFilt4, srcRegFilt6));
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3,
_mm_max_epi16(srcRegFilt5, srcRegFilt7));
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1,
_mm256_castsi256_si128(addFilterReg64));
srcRegFilt3 = _mm_adds_epi16(srcRegFilt3,
_mm256_castsi256_si128(addFilterReg64));
// shift by 7 bit each 16 bit
srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7);
srcRegFilt3 = _mm_srai_epi16(srcRegFilt3, 7);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt3);
// save 16 bytes
_mm_store_si128((__m128i*)output_ptr, srcRegFilt1);
}
}
void Viterbi::AlignWithOutCellOff(HMMSimd* q, HMMSimd* t,ViterbiMatrix * viterbiMatrix,
int maxres, ViterbiResult* result)
#endif
#endif
{
// Linear topology of query (and template) HMM:
// 1. The HMM HMM has L+2 columns. Columns 1 to L contain
// a match state, a delete state and an insert state each.
// 2. The Start state is M0, the virtual match state in column i=0 (j=0). (Therefore X[k][0]=ANY)
// This column has only a match state and it has only a transitions to the next match state.
// 3. The End state is M(L+1), the virtual match state in column i=L+1.(j=L+1) (Therefore X[k][L+1]=ANY)
// Column L has no transitions to the delete state: tr[L][M2D]=tr[L][D2D]=0.
// 4. Transitions I->D and D->I are ignored, since they do not appear in PsiBlast alignments
// (as long as the gap opening penalty d is higher than the best match score S(a,b)).
// Pairwise alignment of two HMMs:
// 1. Pair-states for the alignment of two HMMs are
// MM (Q:Match T:Match) , GD (Q:Gap T:Delete), IM (Q:Insert T:Match), DG (Q:Delelte, T:Match) , MI (Q:Match T:Insert)
// 2. Transitions are allowed only between the MM-state and each of the four other states.
// Saving space:
// The best score ending in pair state XY sXY[i][j] is calculated from left to right (j=1->t->L)
// and top to bottom (i=1->q->L). To save space, only the last row of scores calculated is kept in memory.
// (The backtracing matrices are kept entirely in memory [O(t->L*q->L)]).
// When the calculation has proceeded up to the point where the scores for cell (i,j) are caculated,
// sXY[i-1][j'] = sXY[j'] for j'>=j (A below)
// sXY[i][j'] = sXY[j'] for j'<j (B below)
// sXY[i-1][j-1]= sXY_i_1_j_1 (C below)
// sXY[i][j] = sXY_i_j (D below)
// j-1
// j
// i-1: CAAAAAAAAAAAAAAAAAA
// i : BBBBBBBBBBBBBD
// Variable declarations
const float smin = (this->local ? 0 : -FLT_MAX); //used to distinguish between SW and NW algorithms in maximization
const simd_float smin_vec = simdf32_set(smin);
const simd_float shift_vec = simdf32_set(shift);
// const simd_float one_vec = simdf32_set(1); // 00000001
const simd_int mm_vec = simdi32_set(2); //MM 00000010
const simd_int gd_vec = simdi32_set(3); //GD 00000011
const simd_int im_vec = simdi32_set(4); //IM 00000100
const simd_int dg_vec = simdi32_set(5); //DG 00000101
const simd_int mi_vec = simdi32_set(6); //MI 00000110
const simd_int gd_mm_vec = simdi32_set(8); // 00001000
const simd_int im_mm_vec = simdi32_set(16);// 00010000
const simd_int dg_mm_vec = simdi32_set(32);// 00100000
const simd_int mi_mm_vec = simdi32_set(64);// 01000000
#ifdef VITERBI_SS_SCORE
HMM * q_s = q->GetHMM(0);
const unsigned char * t_index;
if(ss_hmm_mode == HMM::PRED_PRED || ss_hmm_mode == HMM::DSSP_PRED ){
t_index = t->pred_index;
}else if(ss_hmm_mode == HMM::PRED_DSSP){
t_index = t->dssp_index;
}
simd_float * ss_score_vec = (simd_float *) ss_score;
#endif
#ifdef AVX2
const simd_int shuffle_mask_extract = _mm256_setr_epi8(0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, 0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1);
#endif
#ifdef VITERBI_CELLOFF
const __m128i tmp_vec = _mm_set_epi32(0x40000000,0x00400000,0x00004000,0x00000040);//01000000010000000100000001000000
#ifdef AVX2
const simd_int co_vec = _mm256_inserti128_si256(_mm256_castsi128_si256(tmp_vec), tmp_vec, 1);
const simd_int float_min_vec = (simd_int) _mm256_set1_ps(-FLT_MAX);
const simd_int shuffle_mask_celloff = _mm256_set_epi8(
15, 14, 13, 12,
15, 14, 13, 12,
15, 14, 13, 12,
15, 14, 13, 12,
3, 2, 1, 0,
3, 2, 1, 0,
3, 2, 1, 0,
3, 2, 1, 0);
#else // SSE case
const simd_int co_vec = tmp_vec;
const simd_int float_min_vec = (simd_int) simdf32_set(-FLT_MAX);
#endif
#endif // AVX2 end
int i,j; //query and template match state indices
simd_int i2_vec = simdi32_set(0);
simd_int j2_vec = simdi32_set(0);
simd_float sMM_i_j = simdf32_set(0);
simd_float sMI_i_j,sIM_i_j,sGD_i_j,sDG_i_j;
simd_float Si_vec;
simd_float sMM_i_1_j_1;
simd_float sMI_i_1_j_1;
simd_float sIM_i_1_j_1;
simd_float sGD_i_1_j_1;
simd_float sDG_i_1_j_1;
simd_float score_vec = simdf32_set(-FLT_MAX);
simd_int byte_result_vec = simdi32_set(0);
// Initialization of top row, i.e. cells (0,j)
for (j=0; j <= t->L; ++j)
{
const unsigned int index_pos_j = j * 5;
sMM_DG_MI_GD_IM_vec[index_pos_j + 0] = simdf32_set(-j*penalty_gap_template);
sMM_DG_MI_GD_IM_vec[index_pos_j + 1] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_j + 2] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_j + 3] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_j + 4] = simdf32_set(-FLT_MAX);
}
// Viterbi algorithm
const int queryLength = q->L;
for (i=1; i <= queryLength; ++i) // Loop through query positions i
{
// If q is compared to t, exclude regions where overlap of q with t < min_overlap residues
// Initialize cells
sMM_i_1_j_1 = simdf32_set(-(i - 1) * penalty_gap_query); // initialize at (i-1,0)
sIM_i_1_j_1 = simdf32_set(-FLT_MAX); // initialize at (i-1,jmin-1)
sMI_i_1_j_1 = simdf32_set(-FLT_MAX);
sDG_i_1_j_1 = simdf32_set(-FLT_MAX);
sGD_i_1_j_1 = simdf32_set(-FLT_MAX);
// initialize at (i,jmin-1)
const unsigned int index_pos_i = 0 * 5;
sMM_DG_MI_GD_IM_vec[index_pos_i + 0] = simdf32_set(-i * penalty_gap_query); // initialize at (i,0)
sMM_DG_MI_GD_IM_vec[index_pos_i + 1] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_i + 2] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_i + 3] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_i + 4] = simdf32_set(-FLT_MAX);
#ifdef AVX2
unsigned long long * sCO_MI_DG_IM_GD_MM_vec = (unsigned long long *) viterbiMatrix->getRow(i);
#else
unsigned int *sCO_MI_DG_IM_GD_MM_vec = (unsigned int *) viterbiMatrix->getRow(i);
#endif
const unsigned int start_pos_tr_i_1 = (i - 1) * 7;
const unsigned int start_pos_tr_i = (i) * 7;
const simd_float q_m2m = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 2)); // M2M
const simd_float q_m2d = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 3)); // M2D
const simd_float q_d2m = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 4)); // D2M
const simd_float q_d2d = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 5)); // D2D
const simd_float q_i2m = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 6)); // I2m
const simd_float q_i2i = simdf32_load((float *) (q->tr + start_pos_tr_i)); // I2I
const simd_float q_m2i = simdf32_load((float *) (q->tr + start_pos_tr_i + 1)); // M2I
// Find maximum score; global alignment: maxize only over last row and last column
const bool findMaxInnerLoop = (local || i == queryLength);
const int targetLength = t->L;
#ifdef VITERBI_SS_SCORE
if(ss_hmm_mode == HMM::NO_SS_INFORMATION){
// set all to log(1.0) = 0.0
memset(ss_score, 0, (targetLength+1)*VECSIZE_FLOAT*sizeof(float));
}else {
const float * score;
if(ss_hmm_mode == HMM::PRED_PRED){
score = &S33[ (int)q_s->ss_pred[i]][ (int)q_s->ss_conf[i]][0][0];
}else if (ss_hmm_mode == HMM::DSSP_PRED){
score = &S73[ (int)q_s->ss_dssp[i]][0][0];
}else{
score = &S37[ (int)q_s->ss_pred[i]][ (int)q_s->ss_conf[i]][0];
}
// access SS scores and write them to the ss_score array
for (j = 0; j <= (targetLength*VECSIZE_FLOAT); j++) // Loop through template positions j
{
ss_score[j] = ssw * score[t_index[j]];
}
}
#endif
for (j=1; j <= targetLength; ++j) // Loop through template positions j
{
simd_int index_vec;
simd_int res_gt_vec;
// cache line optimized reading
const unsigned int start_pos_tr_j_1 = (j-1) * 7;
const unsigned int start_pos_tr_j = (j) * 7;
const simd_float t_m2m = simdf32_load((float *) (t->tr+start_pos_tr_j_1+2)); // M2M
const simd_float t_m2d = simdf32_load((float *) (t->tr+start_pos_tr_j_1+3)); // M2D
const simd_float t_d2m = simdf32_load((float *) (t->tr+start_pos_tr_j_1+4)); // D2M
const simd_float t_d2d = simdf32_load((float *) (t->tr+start_pos_tr_j_1+5)); // D2D
const simd_float t_i2m = simdf32_load((float *) (t->tr+start_pos_tr_j_1+6)); // I2m
const simd_float t_i2i = simdf32_load((float *) (t->tr+start_pos_tr_j)); // I2i
const simd_float t_m2i = simdf32_load((float *) (t->tr+start_pos_tr_j+1)); // M2I
// Find max value
// CALCULATE_MAX6( sMM_i_j,
// smin,
// sMM_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][M2M],
// sGD_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][D2M],
// sIM_i_1_j_1 + q->tr[i-1][I2M] + t->tr[j-1][M2M],
// sDG_i_1_j_1 + q->tr[i-1][D2M] + t->tr[j-1][M2M],
// sMI_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][I2M],
// bMM[i][j]
// );
// same as sMM_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][M2M]
simd_float mm_m2m_m2m_vec = simdf32_add( simdf32_add(sMM_i_1_j_1, q_m2m), t_m2m);
// if mm > min { 2 }
res_gt_vec = (simd_int)simdf32_gt(mm_m2m_m2m_vec, smin_vec);
byte_result_vec = simdi_and(res_gt_vec, mm_vec);
sMM_i_j = simdf32_max(smin_vec, mm_m2m_m2m_vec);
// same as sGD_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][D2M]
simd_float gd_m2m_d2m_vec = simdf32_add( simdf32_add(sGD_i_1_j_1, q_m2m), t_d2m);
// if gd > max { 3 }
res_gt_vec = (simd_int)simdf32_gt(gd_m2m_d2m_vec, sMM_i_j);
index_vec = simdi_and( res_gt_vec, gd_vec);
byte_result_vec = simdi_or( index_vec, byte_result_vec);
sMM_i_j = simdf32_max(sMM_i_j, gd_m2m_d2m_vec);
// same as sIM_i_1_j_1 + q->tr[i-1][I2M] + t->tr[j-1][M2M]
simd_float im_m2m_d2m_vec = simdf32_add( simdf32_add(sIM_i_1_j_1, q_i2m), t_m2m);
// if im > max { 4 }
MAX2(im_m2m_d2m_vec, sMM_i_j, im_vec,byte_result_vec);
sMM_i_j = simdf32_max(sMM_i_j, im_m2m_d2m_vec);
// same as sDG_i_1_j_1 + q->tr[i-1][D2M] + t->tr[j-1][M2M]
simd_float dg_m2m_d2m_vec = simdf32_add( simdf32_add(sDG_i_1_j_1, q_d2m), t_m2m);
// if dg > max { 5 }
MAX2(dg_m2m_d2m_vec, sMM_i_j, dg_vec,byte_result_vec);
sMM_i_j = simdf32_max(sMM_i_j, dg_m2m_d2m_vec);
// same as sMI_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][I2M],
simd_float mi_m2m_d2m_vec = simdf32_add( simdf32_add(sMI_i_1_j_1, q_m2m), t_i2m);
// if mi > max { 6 }
MAX2(mi_m2m_d2m_vec, sMM_i_j, mi_vec, byte_result_vec);
sMM_i_j = simdf32_max(sMM_i_j, mi_m2m_d2m_vec);
// TODO add secondary structure score
// calculate amino acid profile-profile scores
Si_vec = log2f4(ScalarProd20Vec((simd_float *) q->p[i],(simd_float *) t->p[j]));
#ifdef VITERBI_SS_SCORE
Si_vec = simdf32_add(ss_score_vec[j], Si_vec);
#endif
Si_vec = simdf32_add(Si_vec, shift_vec);
sMM_i_j = simdf32_add(sMM_i_j, Si_vec);
//+ ScoreSS(q,t,i,j) + shift + (Sstruc==NULL? 0: Sstruc[i][j]);
const unsigned int index_pos_j = (j * 5);
const unsigned int index_pos_j_1 = (j - 1) * 5;
const simd_float sMM_j_1 = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j_1 + 0));
const simd_float sGD_j_1 = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j_1 + 3));
const simd_float sIM_j_1 = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j_1 + 4));
const simd_float sMM_j = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j + 0));
const simd_float sDG_j = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j + 1));
const simd_float sMI_j = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j + 2));
sMM_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 0));
sDG_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 1));
sMI_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 2));
sGD_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 3));
sIM_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 4));
// sGD_i_j = max2
// (
// sMM[j-1] + t->tr[j-1][M2D], // MM->GD gap opening in query
// sGD[j-1] + t->tr[j-1][D2D], // GD->GD gap extension in query
// bGD[i][j]
// );
//sMM_DG_GD_MI_IM_vec
simd_float mm_gd_vec = simdf32_add(sMM_j_1, t_m2d); // MM->GD gap opening in query
simd_float gd_gd_vec = simdf32_add(sGD_j_1, t_d2d); // GD->GD gap extension in query
// if mm_gd > gd_dg { 8 }
MAX2_SET_MASK(mm_gd_vec, gd_gd_vec,gd_mm_vec, byte_result_vec);
sGD_i_j = simdf32_max(
mm_gd_vec,
gd_gd_vec
);
// sIM_i_j = max2
// (
// sMM[j-1] + q->tr[i][M2I] + t->tr[j-1][M2M] ,
// sIM[j-1] + q->tr[i][I2I] + t->tr[j-1][M2M], // IM->IM gap extension in query
// bIM[i][j]
// );
simd_float mm_mm_vec = simdf32_add(simdf32_add(sMM_j_1, q_m2i), t_m2m);
simd_float im_im_vec = simdf32_add(simdf32_add(sIM_j_1, q_i2i), t_m2m); // IM->IM gap extension in query
// if mm_mm > im_im { 16 }
MAX2_SET_MASK(mm_mm_vec,im_im_vec, im_mm_vec, byte_result_vec);
sIM_i_j = simdf32_max(
mm_mm_vec,
im_im_vec
);
// sDG_i_j = max2
// (
// sMM[j] + q->tr[i-1][M2D],
// sDG[j] + q->tr[i-1][D2D], //gap extension (DD) in query
// bDG[i][j]
// );
simd_float mm_dg_vec = simdf32_add(sMM_j, q_m2d);
simd_float dg_dg_vec = simdf32_add(sDG_j, q_d2d); //gap extension (DD) in query
// if mm_dg > dg_dg { 32 }
MAX2_SET_MASK(mm_dg_vec,dg_dg_vec, dg_mm_vec, byte_result_vec);
sDG_i_j = simdf32_max( mm_dg_vec
,
dg_dg_vec
);
// sMI_i_j = max2
// (
// sMM[j] + q->tr[i-1][M2M] + t->tr[j][M2I], // MM->MI gap opening M2I in template
// sMI[j] + q->tr[i-1][M2M] + t->tr[j][I2I], // MI->MI gap extension I2I in template
// bMI[i][j]
// );
simd_float mm_mi_vec = simdf32_add( simdf32_add(sMM_j, q_m2m), t_m2i); // MM->MI gap opening M2I in template
simd_float mi_mi_vec = simdf32_add( simdf32_add(sMI_j, q_m2m), t_i2i); // MI->MI gap extension I2I in template
// if mm_mi > mi_mi { 64 }
MAX2_SET_MASK(mm_mi_vec, mi_mi_vec,mi_mm_vec, byte_result_vec);
sMI_i_j = simdf32_max(
mm_mi_vec,
mi_mi_vec
);
// Cell of logic
// if (cell_off[i][j])
//shift 10000000100000001000000010000000 -> 01000000010000000100000001000000
//because 10000000000000000000000000000000 = -2147483648 kills cmplt
#ifdef VITERBI_CELLOFF
#ifdef AVX2
simd_int matrix_vec = _mm256_set1_epi64x(sCO_MI_DG_IM_GD_MM_vec[j]>>1);
matrix_vec = _mm256_shuffle_epi8(matrix_vec,shuffle_mask_celloff);
#else
// if(((sCO_MI_DG_IM_GD_MM_vec[j] >>1) & 0x40404040) > 0){
// std::cout << ((sCO_MI_DG_IM_GD_MM_vec[j] >>1) & 0x40404040 ) << std::endl;
// }
simd_int matrix_vec = simdi32_set(sCO_MI_DG_IM_GD_MM_vec[j]>>1);
#endif
simd_int cell_off_vec = simdi_and(matrix_vec, co_vec);
simd_int res_eq_co_vec = simdi32_gt(co_vec, cell_off_vec ); // shift is because signed can't be checked here
simd_float cell_off_float_min_vec = (simd_float) simdi_andnot(res_eq_co_vec, float_min_vec); // inverse
sMM_i_j = simdf32_add(sMM_i_j,cell_off_float_min_vec); // add the cell off vec to sMM_i_j. Set -FLT_MAX to cell off
sGD_i_j = simdf32_add(sGD_i_j,cell_off_float_min_vec);
sIM_i_j = simdf32_add(sIM_i_j,cell_off_float_min_vec);
sDG_i_j = simdf32_add(sDG_i_j,cell_off_float_min_vec);
sMI_i_j = simdf32_add(sMI_i_j,cell_off_float_min_vec);
#endif
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 0), sMM_i_j);
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 1), sDG_i_j);
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 2), sMI_i_j);
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 3), sGD_i_j);
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 4), sIM_i_j);
// write values back to ViterbiMatrix
#ifdef AVX2
/* byte_result_vec 000H 000G 000F 000E 000D 000C 000B 000A */
/* abcdefgh 0000 0000 HGFE 0000 0000 0000 0000 DCBA */
const __m256i abcdefgh = _mm256_shuffle_epi8(byte_result_vec, shuffle_mask_extract);
/* abcd 0000 0000 0000 DCBA */
const __m128i abcd = _mm256_castsi256_si128(abcdefgh);
/* efgh 0000 0000 HGFE 0000 */
const __m128i efgh = _mm256_extracti128_si256(abcdefgh, 1);
_mm_storel_epi64((__m128i*)&sCO_MI_DG_IM_GD_MM_vec[j], _mm_or_si128(abcd, efgh));
#else
byte_result_vec = _mm_packs_epi32(byte_result_vec, byte_result_vec);
byte_result_vec = _mm_packus_epi16(byte_result_vec, byte_result_vec);
int int_result = _mm_cvtsi128_si32(byte_result_vec);
sCO_MI_DG_IM_GD_MM_vec[j] = int_result;
#endif
// Find maximum score; global alignment: maxize only over last row and last column
// if(sMM_i_j>score && (par.loc || i==q->L)) { i2=i; j2=j; score=sMM_i_j; }
if (findMaxInnerLoop){
// new score is higer
// output
// 0 0 0 MAX
simd_int lookup_mask_hi = (simd_int) simdf32_gt(sMM_i_j,score_vec);
// old score is higher
// output
// MAX MAX MAX 0
simd_int lookup_mask_lo = (simd_int) simdf32_lt(sMM_i_j,score_vec);
simd_int curr_pos_j = simdi32_set(j);
simd_int new_j_pos_hi = simdi_and(lookup_mask_hi,curr_pos_j);
simd_int old_j_pos_lo = simdi_and(lookup_mask_lo,j2_vec);
j2_vec = simdi32_add(new_j_pos_hi,old_j_pos_lo);
simd_int curr_pos_i = simdi32_set(i);
simd_int new_i_pos_hi = simdi_and(lookup_mask_hi,curr_pos_i);
simd_int old_i_pos_lo = simdi_and(lookup_mask_lo,i2_vec);
i2_vec = simdi32_add(new_i_pos_hi,old_i_pos_lo);
score_vec=simdf32_max(sMM_i_j,score_vec);
}
} //end for j
// if global alignment: look for best cell in last column
if (!local){
// new score is higer
// output
// 0 0 0 MAX
simd_int lookup_mask_hi = (simd_int) simdf32_gt(sMM_i_j,score_vec);
// old score is higher
// output
// MAX MAX MAX 0
simd_int lookup_mask_lo = (simd_int) simdf32_lt(sMM_i_j,score_vec);
simd_int curr_pos_j = simdi32_set(j);
simd_int new_j_pos_hi = simdi_and(lookup_mask_hi,curr_pos_j);
simd_int old_j_pos_lo = simdi_and(lookup_mask_lo,j2_vec);
j2_vec = simdi32_add(new_j_pos_hi,old_j_pos_lo);
simd_int curr_pos_i = simdi32_set(i);
simd_int new_i_pos_hi = simdi_and(lookup_mask_hi,curr_pos_i);
simd_int old_i_pos_lo = simdi_and(lookup_mask_lo,i2_vec);
i2_vec = simdi32_add(new_i_pos_hi,old_i_pos_lo);
score_vec = simdf32_max(sMM_i_j,score_vec);
} // end for j
} // end for i
for(int seq_index=0; seq_index < maxres; seq_index++){
result->score[seq_index]=((float*)&score_vec)[seq_index];
result->i[seq_index] = ((int*)&i2_vec)[seq_index];
result->j[seq_index] = ((int*)&j2_vec)[seq_index];
// std::cout << seq_index << "\t" << result->score[seq_index] << "\t" << result->i[seq_index] <<"\t" << result->j[seq_index] << std::endl;
}
// printf("Template=%-12.12s i=%-4i j=%-4i score=%6.3f\n",t->name,i2,j2,score);
}
static INLINE __m256i xx_loadu2_m128i(const void *hi, const void *lo) {
__m128i a0 = _mm_lddqu_si128((const __m128i *)(lo));
__m128i a1 = _mm_lddqu_si128((const __m128i *)(hi));
__m256i a = _mm256_castsi128_si256(a0);
return _mm256_inserti128_si256(a, a1, 1);
}
// Immediate should be truncated to one bit.
__m256i test_mm256_inserti128_si256_2(__m256i a, __m128i b) {
// CHECK-LABEL: @test_mm256_inserti128_si256_2
// CHECK: shufflevector{{.*}}<i32 4, i32 5, i32 2, i32 3>
return _mm256_inserti128_si256(a, b, 2);
}
__m256i test_mm256_inserti128_si256_1(__m256i a, __m128i b) {
// CHECK-LABEL: @test_mm256_inserti128_si256_1
// CHECK: shufflevector{{.*}}<i32 0, i32 1, i32 4, i32 5>
return _mm256_inserti128_si256(a, b, 1);
}
_mm256_loadu2_m128i(const __m128i* const hiaddr, const __m128i* const loaddr)
{
return _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128(loaddr)), _mm_loadu_si128(hiaddr), 1);
}
void extern
avx2_test (void)
{
x = _mm256_inserti128_si256 (x, y, 1);
}
void
test1bit (void)
{
i1 = _mm256_extracti128_si256 (l1, 2); /* { dg-error "the last argument must be an 1-bit immediate" } */
l1 = _mm256_inserti128_si256 (l1, i2, 2); /* { dg-error "the last argument must be an 1-bit immediate" } */
}
