void static
avx2_test (void)
{
union256i_b u, s1, s2;
unsigned char e[32];
unsigned i, tmp;
s1.x = _mm256_set_epi8 (1, 2, 3, 4, 10, 20, 30, 90, 80, 40, 100, 15,
98, 25, 98, 7, 88, 44, 33, 22, 11, 98, 76,
200, 34, 78, 39, 6, 3, 4, 5, 119);
s2.x = _mm256_set_epi8 (88, 44, 33, 220, 11, 98, 76, 100, 34, 78, 39,
6, 3, 4, 5, 219, 1, 2, 3, 4, 10, 20, 30, 90,
80, 40, 100, 15, 98, 25, 98, 7);
u.x = _mm256_adds_epu8 (s1.x, s2.x);
for (i = 0; i < 32; i++)
{
tmp = (unsigned char) s1.a[i] + (unsigned char) s2.a[i];
if (tmp > 255)
tmp = 255;
e[i] = tmp;
}
if (check_union256i_b (u, e))
abort ();
}
void key_schedule(const unsigned char *k, u256 rk[40][16]) {
int i, j;
u256 tk1[32], tmp[32];
unsigned char *tmp_key = malloc(32);
for(i = 0; i < 2; i++)
memcpy(tmp_key + 16*i, k, 16);
pack_key(tk1, tmp_key);
for(j = 0; j < 40; j++) {
//Extract round key
for(i = 0; i < 16; i++){
rk[j][i] = tk1[i];
}
//Add constant into key
u256 rc = _mm256_set_epi64x(0x000000FF000000FFull,
0x000000FF000000FFull,
0x000000FF000000FFull,
0x000000FF000000FFull);
if(RC[j]>>5 & 1)
rk[j][14] = XOR(rk[j][14], rc);
if(RC[j]>>4 & 1)
rk[j][15] = XOR(rk[j][15], rc);
if(RC[j]>>3 & 1)
rk[j][4] = XOR(rk[j][4], rc);
if(RC[j]>>2 & 1)
rk[j][5] = XOR(rk[j][5], rc);
if(RC[j]>>1 & 1)
rk[j][6] = XOR(rk[j][6], rc);
if(RC[j]>>0 & 1)
rk[j][7] = XOR(rk[j][7], rc);
//Update TK1
for(i = 0; i < 16; i++){
tmp[16 + i] = tk1[0 + i];
}
//Apply bit permutation
for(i = 0; i < 8; i++){
tmp[0 + i] = XOR(_mm256_shuffle_epi8(tk1[16 + i], _mm256_set_epi8(0xff,28,0xff,29,0xff,24,0xff,25,0xff,20,0xff,21,0xff,16,0xff,17,0xff,12,0xff,13,0xff,8,0xff,9,0xff,4,0xff,5,0xff,0,0xff,1)),
_mm256_shuffle_epi8(tk1[24 + i], _mm256_set_epi8(29,0xff,31,0xff,25,0xff,27,0xff,21,0xff,23,0xff,17,0xff,19,0xff,13,0xff,15,0xff,9,0xff,11,0xff,5,0xff,7,0xff,1,0xff,3,0xff)));
tmp[8 + i] = XOR(_mm256_shuffle_epi8(tk1[16 + i], _mm256_set_epi8(31,0xff,0xff,30,27,0xff,0xff,26,23,0xff,0xff,22,19,0xff,0xff,18,15,0xff,0xff,14,11,0xff,0xff,10,7,0xff,0xff,6,3,0xff,0xff,2)),
_mm256_shuffle_epi8(tk1[24 + i], _mm256_set_epi8(0xff,28,30,0xff,0xff,24,26,0xff,0xff,20,22,0xff,0xff,16,18,0xff,0xff,12,14,0xff,0xff,8,10,0xff,0xff,4,6,0xff,0xff,0,2,0xff)));
}
for(i = 0; i < 32; i++){
tk1[i] = tmp[i];
}
}
free(tmp_key);
}
/* Routine optimized for shuffling a buffer for a type size of 16 bytes. */
static void
shuffle16_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements)
{
static const size_t bytesoftype = 16;
size_t j;
int k, l;
__m256i ymm0[16], ymm1[16];
/* Create the shuffle mask.
NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from
most to least significant (i.e., their order is reversed when compared to
loading the mask from an array). */
const __m256i shmask = _mm256_set_epi8(
0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04,
0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00,
0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04,
0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00);
for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) {
/* Fetch 32 elements (512 bytes) into 16 YMM registers. */
for (k = 0; k < 16; k++) {
ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i))));
}
/* Transpose bytes */
for (k = 0, l = 0; k < 8; k++, l +=2) {
ymm1[k*2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l+1]);
ymm1[k*2+1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l+1]);
}
/* Transpose words */
for (k = 0, l = -2; k < 8; k++, l++) {
if ((k%2) == 0) l += 2;
ymm0[k*2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l+2]);
ymm0[k*2+1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l+2]);
}
/* Transpose double words */
for (k = 0, l = -4; k < 8; k++, l++) {
if ((k%4) == 0) l += 4;
ymm1[k*2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l+4]);
ymm1[k*2+1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l+4]);
}
/* Transpose quad words */
for (k = 0; k < 8; k++) {
ymm0[k*2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k+8]);
ymm0[k*2+1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k+8]);
}
for (k = 0; k < 16; k++) {
ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8);
ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask);
}
/* Store the result vectors */
uint8_t* const dest_for_jth_element = dest + j;
for (k = 0; k < 16; k++) {
_mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm0[k]);
}
}
}
// Reverse with intrinsics
// First - reverse the values in the first half and second half
// Second - copy the second half of temp array to the beginning of original one, and first half (from temp array) next to the first in original one
// (swap the halfs)
inline void reverse(char * bytes, int numChunks)
{
static char temp[64];
__m256i firstHalf, secondHalf;
for (int i = 0; i < numChunks; ++i)
{
// Reverse first half.
firstHalf = _mm256_set_epi8(bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15],
bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21], bytes[22], bytes[23], bytes[24], bytes[25], bytes[26], bytes[27], bytes[28], bytes[29], bytes[30], bytes[31]);
// Reverse second half.
secondHalf = _mm256_set_epi8(bytes[32], bytes[33], bytes[34], bytes[35], bytes[36], bytes[37], bytes[38], bytes[39], bytes[40], bytes[41], bytes[42], bytes[43], bytes[44], bytes[45], bytes[46], bytes[47], bytes[48],
bytes[49], bytes[50], bytes[51], bytes[52], bytes[53], bytes[54], bytes[55], bytes[56], bytes[57], bytes[58], bytes[59], bytes[60], bytes[61], bytes[62], bytes[63]);
// write the second half at the begining, and after it- first one.
_mm256_storeu_si256((__m256i*)bytes, secondHalf);
_mm256_storeu_si256((__m256i*)(bytes + 32), firstHalf);
bytes += 64;
}
}
void static
avx2_test (void)
{
union256i_b u, s1, s2;
char e[32];
unsigned i;
s1.x = _mm256_set_epi8 (10, 74, 50, 4, 6, 99, 1, 4, 87, 83, 84,
29, 81, 79, 1, 3, 1, 5, 2, 47, 20, 2, 72,
92, 9, 4, 23, 17, 99, 43, 72, 17);
s2.x = _mm256_set_epi8 (88, 44, 33, 20, 56, 99, 2, 90, 38, 4, 200,
17, 3, 39, 2, 37, 27, 95, 17, 74, 72, 43,
27, 112, 71, 50, 32, 72, 84, 17, 27, 96);
u.x = _mm256_add_epi8 (s1.x, s2.x);
for (i = 0; i < 32; i++)
e[i] = s1.a[i] + s2.a[i];
if (check_union256i_b (u, e))
abort ();
}
inline void reverse(char* bytes, int numChunks)
{
static __m256i reversed1;
static __m256i reversed2;
for(size_t i = 0; i < numChunks; ++i)
{
reversed1 = _mm256_set_epi8(bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8],
bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], bytes[16],
bytes[17], bytes[18], bytes[19], bytes[20], bytes[21], bytes[22], bytes[23], bytes[24],
bytes[25], bytes[26], bytes[27], bytes[28], bytes[29], bytes[30], bytes[31]);
reversed2 = _mm256_set_epi8(bytes[32], bytes[33], bytes[34], bytes[35], bytes[36], bytes[37], bytes[38], bytes[39],
bytes[40], bytes[41], bytes[42], bytes[43], bytes[44], bytes[45], bytes[46], bytes[47],
bytes[48], bytes[49], bytes[50], bytes[51], bytes[52], bytes[53], bytes[54], bytes[55],
bytes[56], bytes[57], bytes[58], bytes[59], bytes[60], bytes[61], bytes[62], bytes[63]);
_mm256_storeu_si256((__m256i*)&bytes[0], reversed2);
_mm256_storeu_si256((__m256i*)&bytes[32], reversed1);
bytes += 64;
}
}
void static
avx2_test (void)
{
union256i_b u, s1, s2;
unsigned char e[32];
int tmp;
int i;
s1.x = _mm256_set_epi8 (1, 2, 3, 4, 10, 20, 30, 90, -80, -40, -100,
-15, 98, 25, 98, 7, 88, 44, 33, 22, 11, 98,
76, -100, -34, -78, -39, 6, 3, 4, 5, 119);
s2.x = _mm256_set_epi8 (88, 44, 33, 22, 11, 98, 76, -100, -34, -78,
-39, 6, 3, 4, 5, 119, 1, 2, 3, 4, 10, 20,
30, 90, -80, -40, -100, -15, 98, 25, 98, 7);
u.x = _mm256_avg_epu8 (s1.x, s2.x);
for (i = 0; i < 32; i++)
e[i] = ((unsigned char) s1.a[i] + (unsigned char) s2.a[i] + 1) >> 1;
if (check_union256i_b (u, e))
abort ();
}
static inline __m256i
enc_reshuffle (__m256i in)
{
// Spread out 32-bit words over both halves of the input register:
in = _mm256_permutevar8x32_epi32(in, _mm256_setr_epi32(
0, 1, 2, -1,
3, 4, 5, -1));
// Slice into 32-bit chunks and operate on all chunks in parallel.
// All processing is done within the 32-bit chunk. First, shuffle:
// before: [eeeeeeff|ccdddddd|bbbbcccc|aaaaaabb]
// after: [00000000|aaaaaabb|bbbbcccc|ccdddddd]
in = _mm256_shuffle_epi8(in, _mm256_set_epi8(
-1, 9, 10, 11,
-1, 6, 7, 8,
-1, 3, 4, 5,
-1, 0, 1, 2,
-1, 9, 10, 11,
-1, 6, 7, 8,
-1, 3, 4, 5,
-1, 0, 1, 2));
// cd = [00000000|00000000|0000cccc|ccdddddd]
const __m256i cd = _mm256_and_si256(in, _mm256_set1_epi32(0x00000FFF));
// ab = [0000aaaa|aabbbbbb|00000000|00000000]
const __m256i ab = _mm256_and_si256(_mm256_slli_epi32(in, 4), _mm256_set1_epi32(0x0FFF0000));
// merged = [0000aaaa|aabbbbbb|0000cccc|ccdddddd]
const __m256i merged = _mm256_or_si256(ab, cd);
// bd = [00000000|00bbbbbb|00000000|00dddddd]
const __m256i bd = _mm256_and_si256(merged, _mm256_set1_epi32(0x003F003F));
// ac = [00aaaaaa|00000000|00cccccc|00000000]
const __m256i ac = _mm256_and_si256(_mm256_slli_epi32(merged, 2), _mm256_set1_epi32(0x3F003F00));
// indices = [00aaaaaa|00bbbbbb|00cccccc|00dddddd]
const __m256i indices = _mm256_or_si256(ac, bd);
// return = [00dddddd|00cccccc|00bbbbbb|00aaaaaa]
return _mm256_bswap_epi32(indices);
}
/* Routine optimized for shuffling a buffer for a type size of 2 bytes. */
static void
shuffle2_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements)
{
static const size_t bytesoftype = 2;
size_t j;
int k;
__m256i ymm0[2], ymm1[2];
/* Create the shuffle mask.
NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from
most to least significant (i.e., their order is reversed when compared to
loading the mask from an array). */
const __m256i shmask = _mm256_set_epi8(
0x0f, 0x0d, 0x0b, 0x09, 0x07, 0x05, 0x03, 0x01,
0x0e, 0x0c, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x00,
0x0f, 0x0d, 0x0b, 0x09, 0x07, 0x05, 0x03, 0x01,
0x0e, 0x0c, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x00);
for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) {
/* Fetch 32 elements (64 bytes) then transpose bytes, words and double words. */
for (k = 0; k < 2; k++) {
ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i))));
ymm1[k] = _mm256_shuffle_epi8(ymm0[k], shmask);
}
ymm0[0] = _mm256_permute4x64_epi64(ymm1[0], 0xd8);
ymm0[1] = _mm256_permute4x64_epi64(ymm1[1], 0x8d);
ymm1[0] = _mm256_blend_epi32(ymm0[0], ymm0[1], 0xf0);
ymm0[1] = _mm256_blend_epi32(ymm0[0], ymm0[1], 0x0f);
ymm1[1] = _mm256_permute4x64_epi64(ymm0[1], 0x4e);
/* Store the result vectors */
uint8_t* const dest_for_jth_element = dest + j;
for (k = 0; k < 2; k++) {
_mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm1[k]);
}
}
}
static inline void do_encode_12bytes(const char (*alphabet)[2], char *out, __m256i chunk)
{
const __m256i shufflemask = _mm256_set_epi8(
-1, 9, 10, 11,
-1, 9, 10, 11,
-1, 6, 7, 8,
-1, 6, 7, 8,
-1, 3, 4, 5,
-1, 3, 4, 5,
-1, 0, 1, 2,
-1, 0, 1, 2
);
const __m256i shifts = _mm256_set_epi32(0, 12, 0, 12, 0, 12, 0, 12);
const __m256i masks = _mm256_set1_epi32(4095);
// convert from big endian and rearrange the bytes
chunk = _mm256_shuffle_epi8(chunk, shufflemask);
chunk = _mm256_srlv_epi32(chunk, shifts);
chunk = _mm256_and_si256(chunk, masks);
// write the two halves to memory
do_encode_6bytes(alphabet, out + 0, _mm256_extracti128_si256(chunk, 0));
do_encode_6bytes(alphabet, out + 8, _mm256_extracti128_si256(chunk, 1));
}
void Viterbi::AlignWithOutCellOff(HMMSimd* q, HMMSimd* t,ViterbiMatrix * viterbiMatrix,
int maxres, ViterbiResult* result)
#endif
#endif
{
// Linear topology of query (and template) HMM:
// 1. The HMM HMM has L+2 columns. Columns 1 to L contain
// a match state, a delete state and an insert state each.
// 2. The Start state is M0, the virtual match state in column i=0 (j=0). (Therefore X[k][0]=ANY)
// This column has only a match state and it has only a transitions to the next match state.
// 3. The End state is M(L+1), the virtual match state in column i=L+1.(j=L+1) (Therefore X[k][L+1]=ANY)
// Column L has no transitions to the delete state: tr[L][M2D]=tr[L][D2D]=0.
// 4. Transitions I->D and D->I are ignored, since they do not appear in PsiBlast alignments
// (as long as the gap opening penalty d is higher than the best match score S(a,b)).
// Pairwise alignment of two HMMs:
// 1. Pair-states for the alignment of two HMMs are
// MM (Q:Match T:Match) , GD (Q:Gap T:Delete), IM (Q:Insert T:Match), DG (Q:Delelte, T:Match) , MI (Q:Match T:Insert)
// 2. Transitions are allowed only between the MM-state and each of the four other states.
// Saving space:
// The best score ending in pair state XY sXY[i][j] is calculated from left to right (j=1->t->L)
// and top to bottom (i=1->q->L). To save space, only the last row of scores calculated is kept in memory.
// (The backtracing matrices are kept entirely in memory [O(t->L*q->L)]).
// When the calculation has proceeded up to the point where the scores for cell (i,j) are caculated,
// sXY[i-1][j'] = sXY[j'] for j'>=j (A below)
// sXY[i][j'] = sXY[j'] for j'<j (B below)
// sXY[i-1][j-1]= sXY_i_1_j_1 (C below)
// sXY[i][j] = sXY_i_j (D below)
// j-1
// j
// i-1: CAAAAAAAAAAAAAAAAAA
// i : BBBBBBBBBBBBBD
// Variable declarations
const float smin = (this->local ? 0 : -FLT_MAX); //used to distinguish between SW and NW algorithms in maximization
const simd_float smin_vec = simdf32_set(smin);
const simd_float shift_vec = simdf32_set(shift);
// const simd_float one_vec = simdf32_set(1); // 00000001
const simd_int mm_vec = simdi32_set(2); //MM 00000010
const simd_int gd_vec = simdi32_set(3); //GD 00000011
const simd_int im_vec = simdi32_set(4); //IM 00000100
const simd_int dg_vec = simdi32_set(5); //DG 00000101
const simd_int mi_vec = simdi32_set(6); //MI 00000110
const simd_int gd_mm_vec = simdi32_set(8); // 00001000
const simd_int im_mm_vec = simdi32_set(16);// 00010000
const simd_int dg_mm_vec = simdi32_set(32);// 00100000
const simd_int mi_mm_vec = simdi32_set(64);// 01000000
#ifdef VITERBI_SS_SCORE
HMM * q_s = q->GetHMM(0);
const unsigned char * t_index;
if(ss_hmm_mode == HMM::PRED_PRED || ss_hmm_mode == HMM::DSSP_PRED ){
t_index = t->pred_index;
}else if(ss_hmm_mode == HMM::PRED_DSSP){
t_index = t->dssp_index;
}
simd_float * ss_score_vec = (simd_float *) ss_score;
#endif
#ifdef AVX2
const simd_int shuffle_mask_extract = _mm256_setr_epi8(0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, 0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1);
#endif
#ifdef VITERBI_CELLOFF
const __m128i tmp_vec = _mm_set_epi32(0x40000000,0x00400000,0x00004000,0x00000040);//01000000010000000100000001000000
#ifdef AVX2
const simd_int co_vec = _mm256_inserti128_si256(_mm256_castsi128_si256(tmp_vec), tmp_vec, 1);
const simd_int float_min_vec = (simd_int) _mm256_set1_ps(-FLT_MAX);
const simd_int shuffle_mask_celloff = _mm256_set_epi8(
15, 14, 13, 12,
15, 14, 13, 12,
15, 14, 13, 12,
15, 14, 13, 12,
3, 2, 1, 0,
3, 2, 1, 0,
3, 2, 1, 0,
3, 2, 1, 0);
#else // SSE case
const simd_int co_vec = tmp_vec;
const simd_int float_min_vec = (simd_int) simdf32_set(-FLT_MAX);
#endif
#endif // AVX2 end
int i,j; //query and template match state indices
simd_int i2_vec = simdi32_set(0);
simd_int j2_vec = simdi32_set(0);
simd_float sMM_i_j = simdf32_set(0);
simd_float sMI_i_j,sIM_i_j,sGD_i_j,sDG_i_j;
simd_float Si_vec;
simd_float sMM_i_1_j_1;
simd_float sMI_i_1_j_1;
simd_float sIM_i_1_j_1;
simd_float sGD_i_1_j_1;
simd_float sDG_i_1_j_1;
simd_float score_vec = simdf32_set(-FLT_MAX);
simd_int byte_result_vec = simdi32_set(0);
// Initialization of top row, i.e. cells (0,j)
for (j=0; j <= t->L; ++j)
{
const unsigned int index_pos_j = j * 5;
sMM_DG_MI_GD_IM_vec[index_pos_j + 0] = simdf32_set(-j*penalty_gap_template);
sMM_DG_MI_GD_IM_vec[index_pos_j + 1] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_j + 2] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_j + 3] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_j + 4] = simdf32_set(-FLT_MAX);
}
// Viterbi algorithm
const int queryLength = q->L;
for (i=1; i <= queryLength; ++i) // Loop through query positions i
{
// If q is compared to t, exclude regions where overlap of q with t < min_overlap residues
// Initialize cells
sMM_i_1_j_1 = simdf32_set(-(i - 1) * penalty_gap_query); // initialize at (i-1,0)
sIM_i_1_j_1 = simdf32_set(-FLT_MAX); // initialize at (i-1,jmin-1)
sMI_i_1_j_1 = simdf32_set(-FLT_MAX);
sDG_i_1_j_1 = simdf32_set(-FLT_MAX);
sGD_i_1_j_1 = simdf32_set(-FLT_MAX);
// initialize at (i,jmin-1)
const unsigned int index_pos_i = 0 * 5;
sMM_DG_MI_GD_IM_vec[index_pos_i + 0] = simdf32_set(-i * penalty_gap_query); // initialize at (i,0)
sMM_DG_MI_GD_IM_vec[index_pos_i + 1] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_i + 2] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_i + 3] = simdf32_set(-FLT_MAX);
sMM_DG_MI_GD_IM_vec[index_pos_i + 4] = simdf32_set(-FLT_MAX);
#ifdef AVX2
unsigned long long * sCO_MI_DG_IM_GD_MM_vec = (unsigned long long *) viterbiMatrix->getRow(i);
#else
unsigned int *sCO_MI_DG_IM_GD_MM_vec = (unsigned int *) viterbiMatrix->getRow(i);
#endif
const unsigned int start_pos_tr_i_1 = (i - 1) * 7;
const unsigned int start_pos_tr_i = (i) * 7;
const simd_float q_m2m = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 2)); // M2M
const simd_float q_m2d = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 3)); // M2D
const simd_float q_d2m = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 4)); // D2M
const simd_float q_d2d = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 5)); // D2D
const simd_float q_i2m = simdf32_load((float *) (q->tr + start_pos_tr_i_1 + 6)); // I2m
const simd_float q_i2i = simdf32_load((float *) (q->tr + start_pos_tr_i)); // I2I
const simd_float q_m2i = simdf32_load((float *) (q->tr + start_pos_tr_i + 1)); // M2I
// Find maximum score; global alignment: maxize only over last row and last column
const bool findMaxInnerLoop = (local || i == queryLength);
const int targetLength = t->L;
#ifdef VITERBI_SS_SCORE
if(ss_hmm_mode == HMM::NO_SS_INFORMATION){
// set all to log(1.0) = 0.0
memset(ss_score, 0, (targetLength+1)*VECSIZE_FLOAT*sizeof(float));
}else {
const float * score;
if(ss_hmm_mode == HMM::PRED_PRED){
score = &S33[ (int)q_s->ss_pred[i]][ (int)q_s->ss_conf[i]][0][0];
}else if (ss_hmm_mode == HMM::DSSP_PRED){
score = &S73[ (int)q_s->ss_dssp[i]][0][0];
}else{
score = &S37[ (int)q_s->ss_pred[i]][ (int)q_s->ss_conf[i]][0];
}
// access SS scores and write them to the ss_score array
for (j = 0; j <= (targetLength*VECSIZE_FLOAT); j++) // Loop through template positions j
{
ss_score[j] = ssw * score[t_index[j]];
}
}
#endif
for (j=1; j <= targetLength; ++j) // Loop through template positions j
{
simd_int index_vec;
simd_int res_gt_vec;
// cache line optimized reading
const unsigned int start_pos_tr_j_1 = (j-1) * 7;
const unsigned int start_pos_tr_j = (j) * 7;
const simd_float t_m2m = simdf32_load((float *) (t->tr+start_pos_tr_j_1+2)); // M2M
const simd_float t_m2d = simdf32_load((float *) (t->tr+start_pos_tr_j_1+3)); // M2D
const simd_float t_d2m = simdf32_load((float *) (t->tr+start_pos_tr_j_1+4)); // D2M
const simd_float t_d2d = simdf32_load((float *) (t->tr+start_pos_tr_j_1+5)); // D2D
const simd_float t_i2m = simdf32_load((float *) (t->tr+start_pos_tr_j_1+6)); // I2m
const simd_float t_i2i = simdf32_load((float *) (t->tr+start_pos_tr_j)); // I2i
const simd_float t_m2i = simdf32_load((float *) (t->tr+start_pos_tr_j+1)); // M2I
// Find max value
// CALCULATE_MAX6( sMM_i_j,
// smin,
// sMM_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][M2M],
// sGD_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][D2M],
// sIM_i_1_j_1 + q->tr[i-1][I2M] + t->tr[j-1][M2M],
// sDG_i_1_j_1 + q->tr[i-1][D2M] + t->tr[j-1][M2M],
// sMI_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][I2M],
// bMM[i][j]
// );
// same as sMM_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][M2M]
simd_float mm_m2m_m2m_vec = simdf32_add( simdf32_add(sMM_i_1_j_1, q_m2m), t_m2m);
// if mm > min { 2 }
res_gt_vec = (simd_int)simdf32_gt(mm_m2m_m2m_vec, smin_vec);
byte_result_vec = simdi_and(res_gt_vec, mm_vec);
sMM_i_j = simdf32_max(smin_vec, mm_m2m_m2m_vec);
// same as sGD_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][D2M]
simd_float gd_m2m_d2m_vec = simdf32_add( simdf32_add(sGD_i_1_j_1, q_m2m), t_d2m);
// if gd > max { 3 }
res_gt_vec = (simd_int)simdf32_gt(gd_m2m_d2m_vec, sMM_i_j);
index_vec = simdi_and( res_gt_vec, gd_vec);
byte_result_vec = simdi_or( index_vec, byte_result_vec);
sMM_i_j = simdf32_max(sMM_i_j, gd_m2m_d2m_vec);
// same as sIM_i_1_j_1 + q->tr[i-1][I2M] + t->tr[j-1][M2M]
simd_float im_m2m_d2m_vec = simdf32_add( simdf32_add(sIM_i_1_j_1, q_i2m), t_m2m);
// if im > max { 4 }
MAX2(im_m2m_d2m_vec, sMM_i_j, im_vec,byte_result_vec);
sMM_i_j = simdf32_max(sMM_i_j, im_m2m_d2m_vec);
// same as sDG_i_1_j_1 + q->tr[i-1][D2M] + t->tr[j-1][M2M]
simd_float dg_m2m_d2m_vec = simdf32_add( simdf32_add(sDG_i_1_j_1, q_d2m), t_m2m);
// if dg > max { 5 }
MAX2(dg_m2m_d2m_vec, sMM_i_j, dg_vec,byte_result_vec);
sMM_i_j = simdf32_max(sMM_i_j, dg_m2m_d2m_vec);
// same as sMI_i_1_j_1 + q->tr[i-1][M2M] + t->tr[j-1][I2M],
simd_float mi_m2m_d2m_vec = simdf32_add( simdf32_add(sMI_i_1_j_1, q_m2m), t_i2m);
// if mi > max { 6 }
MAX2(mi_m2m_d2m_vec, sMM_i_j, mi_vec, byte_result_vec);
sMM_i_j = simdf32_max(sMM_i_j, mi_m2m_d2m_vec);
// TODO add secondary structure score
// calculate amino acid profile-profile scores
Si_vec = log2f4(ScalarProd20Vec((simd_float *) q->p[i],(simd_float *) t->p[j]));
#ifdef VITERBI_SS_SCORE
Si_vec = simdf32_add(ss_score_vec[j], Si_vec);
#endif
Si_vec = simdf32_add(Si_vec, shift_vec);
sMM_i_j = simdf32_add(sMM_i_j, Si_vec);
//+ ScoreSS(q,t,i,j) + shift + (Sstruc==NULL? 0: Sstruc[i][j]);
const unsigned int index_pos_j = (j * 5);
const unsigned int index_pos_j_1 = (j - 1) * 5;
const simd_float sMM_j_1 = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j_1 + 0));
const simd_float sGD_j_1 = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j_1 + 3));
const simd_float sIM_j_1 = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j_1 + 4));
const simd_float sMM_j = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j + 0));
const simd_float sDG_j = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j + 1));
const simd_float sMI_j = simdf32_load((float *) (sMM_DG_MI_GD_IM_vec + index_pos_j + 2));
sMM_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 0));
sDG_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 1));
sMI_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 2));
sGD_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 3));
sIM_i_1_j_1 = simdf32_load((float *)(sMM_DG_MI_GD_IM_vec + index_pos_j + 4));
// sGD_i_j = max2
// (
// sMM[j-1] + t->tr[j-1][M2D], // MM->GD gap opening in query
// sGD[j-1] + t->tr[j-1][D2D], // GD->GD gap extension in query
// bGD[i][j]
// );
//sMM_DG_GD_MI_IM_vec
simd_float mm_gd_vec = simdf32_add(sMM_j_1, t_m2d); // MM->GD gap opening in query
simd_float gd_gd_vec = simdf32_add(sGD_j_1, t_d2d); // GD->GD gap extension in query
// if mm_gd > gd_dg { 8 }
MAX2_SET_MASK(mm_gd_vec, gd_gd_vec,gd_mm_vec, byte_result_vec);
sGD_i_j = simdf32_max(
mm_gd_vec,
gd_gd_vec
);
// sIM_i_j = max2
// (
// sMM[j-1] + q->tr[i][M2I] + t->tr[j-1][M2M] ,
// sIM[j-1] + q->tr[i][I2I] + t->tr[j-1][M2M], // IM->IM gap extension in query
// bIM[i][j]
// );
simd_float mm_mm_vec = simdf32_add(simdf32_add(sMM_j_1, q_m2i), t_m2m);
simd_float im_im_vec = simdf32_add(simdf32_add(sIM_j_1, q_i2i), t_m2m); // IM->IM gap extension in query
// if mm_mm > im_im { 16 }
MAX2_SET_MASK(mm_mm_vec,im_im_vec, im_mm_vec, byte_result_vec);
sIM_i_j = simdf32_max(
mm_mm_vec,
im_im_vec
);
// sDG_i_j = max2
// (
// sMM[j] + q->tr[i-1][M2D],
// sDG[j] + q->tr[i-1][D2D], //gap extension (DD) in query
// bDG[i][j]
// );
simd_float mm_dg_vec = simdf32_add(sMM_j, q_m2d);
simd_float dg_dg_vec = simdf32_add(sDG_j, q_d2d); //gap extension (DD) in query
// if mm_dg > dg_dg { 32 }
MAX2_SET_MASK(mm_dg_vec,dg_dg_vec, dg_mm_vec, byte_result_vec);
sDG_i_j = simdf32_max( mm_dg_vec
,
dg_dg_vec
);
// sMI_i_j = max2
// (
// sMM[j] + q->tr[i-1][M2M] + t->tr[j][M2I], // MM->MI gap opening M2I in template
// sMI[j] + q->tr[i-1][M2M] + t->tr[j][I2I], // MI->MI gap extension I2I in template
// bMI[i][j]
// );
simd_float mm_mi_vec = simdf32_add( simdf32_add(sMM_j, q_m2m), t_m2i); // MM->MI gap opening M2I in template
simd_float mi_mi_vec = simdf32_add( simdf32_add(sMI_j, q_m2m), t_i2i); // MI->MI gap extension I2I in template
// if mm_mi > mi_mi { 64 }
MAX2_SET_MASK(mm_mi_vec, mi_mi_vec,mi_mm_vec, byte_result_vec);
sMI_i_j = simdf32_max(
mm_mi_vec,
mi_mi_vec
);
// Cell of logic
// if (cell_off[i][j])
//shift 10000000100000001000000010000000 -> 01000000010000000100000001000000
//because 10000000000000000000000000000000 = -2147483648 kills cmplt
#ifdef VITERBI_CELLOFF
#ifdef AVX2
simd_int matrix_vec = _mm256_set1_epi64x(sCO_MI_DG_IM_GD_MM_vec[j]>>1);
matrix_vec = _mm256_shuffle_epi8(matrix_vec,shuffle_mask_celloff);
#else
// if(((sCO_MI_DG_IM_GD_MM_vec[j] >>1) & 0x40404040) > 0){
// std::cout << ((sCO_MI_DG_IM_GD_MM_vec[j] >>1) & 0x40404040 ) << std::endl;
// }
simd_int matrix_vec = simdi32_set(sCO_MI_DG_IM_GD_MM_vec[j]>>1);
#endif
simd_int cell_off_vec = simdi_and(matrix_vec, co_vec);
simd_int res_eq_co_vec = simdi32_gt(co_vec, cell_off_vec ); // shift is because signed can't be checked here
simd_float cell_off_float_min_vec = (simd_float) simdi_andnot(res_eq_co_vec, float_min_vec); // inverse
sMM_i_j = simdf32_add(sMM_i_j,cell_off_float_min_vec); // add the cell off vec to sMM_i_j. Set -FLT_MAX to cell off
sGD_i_j = simdf32_add(sGD_i_j,cell_off_float_min_vec);
sIM_i_j = simdf32_add(sIM_i_j,cell_off_float_min_vec);
sDG_i_j = simdf32_add(sDG_i_j,cell_off_float_min_vec);
sMI_i_j = simdf32_add(sMI_i_j,cell_off_float_min_vec);
#endif
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 0), sMM_i_j);
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 1), sDG_i_j);
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 2), sMI_i_j);
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 3), sGD_i_j);
simdf32_store((float *)(sMM_DG_MI_GD_IM_vec+index_pos_j + 4), sIM_i_j);
// write values back to ViterbiMatrix
#ifdef AVX2
/* byte_result_vec 000H 000G 000F 000E 000D 000C 000B 000A */
/* abcdefgh 0000 0000 HGFE 0000 0000 0000 0000 DCBA */
const __m256i abcdefgh = _mm256_shuffle_epi8(byte_result_vec, shuffle_mask_extract);
/* abcd 0000 0000 0000 DCBA */
const __m128i abcd = _mm256_castsi256_si128(abcdefgh);
/* efgh 0000 0000 HGFE 0000 */
const __m128i efgh = _mm256_extracti128_si256(abcdefgh, 1);
_mm_storel_epi64((__m128i*)&sCO_MI_DG_IM_GD_MM_vec[j], _mm_or_si128(abcd, efgh));
#else
byte_result_vec = _mm_packs_epi32(byte_result_vec, byte_result_vec);
byte_result_vec = _mm_packus_epi16(byte_result_vec, byte_result_vec);
int int_result = _mm_cvtsi128_si32(byte_result_vec);
sCO_MI_DG_IM_GD_MM_vec[j] = int_result;
#endif
// Find maximum score; global alignment: maxize only over last row and last column
// if(sMM_i_j>score && (par.loc || i==q->L)) { i2=i; j2=j; score=sMM_i_j; }
if (findMaxInnerLoop){
// new score is higer
// output
// 0 0 0 MAX
simd_int lookup_mask_hi = (simd_int) simdf32_gt(sMM_i_j,score_vec);
// old score is higher
// output
// MAX MAX MAX 0
simd_int lookup_mask_lo = (simd_int) simdf32_lt(sMM_i_j,score_vec);
simd_int curr_pos_j = simdi32_set(j);
simd_int new_j_pos_hi = simdi_and(lookup_mask_hi,curr_pos_j);
simd_int old_j_pos_lo = simdi_and(lookup_mask_lo,j2_vec);
j2_vec = simdi32_add(new_j_pos_hi,old_j_pos_lo);
simd_int curr_pos_i = simdi32_set(i);
simd_int new_i_pos_hi = simdi_and(lookup_mask_hi,curr_pos_i);
simd_int old_i_pos_lo = simdi_and(lookup_mask_lo,i2_vec);
i2_vec = simdi32_add(new_i_pos_hi,old_i_pos_lo);
score_vec=simdf32_max(sMM_i_j,score_vec);
}
} //end for j
// if global alignment: look for best cell in last column
if (!local){
// new score is higer
// output
// 0 0 0 MAX
simd_int lookup_mask_hi = (simd_int) simdf32_gt(sMM_i_j,score_vec);
// old score is higher
// output
// MAX MAX MAX 0
simd_int lookup_mask_lo = (simd_int) simdf32_lt(sMM_i_j,score_vec);
simd_int curr_pos_j = simdi32_set(j);
simd_int new_j_pos_hi = simdi_and(lookup_mask_hi,curr_pos_j);
simd_int old_j_pos_lo = simdi_and(lookup_mask_lo,j2_vec);
j2_vec = simdi32_add(new_j_pos_hi,old_j_pos_lo);
simd_int curr_pos_i = simdi32_set(i);
simd_int new_i_pos_hi = simdi_and(lookup_mask_hi,curr_pos_i);
simd_int old_i_pos_lo = simdi_and(lookup_mask_lo,i2_vec);
i2_vec = simdi32_add(new_i_pos_hi,old_i_pos_lo);
score_vec = simdf32_max(sMM_i_j,score_vec);
} // end for j
} // end for i
for(int seq_index=0; seq_index < maxres; seq_index++){
result->score[seq_index]=((float*)&score_vec)[seq_index];
result->i[seq_index] = ((int*)&i2_vec)[seq_index];
result->j[seq_index] = ((int*)&j2_vec)[seq_index];
// std::cout << seq_index << "\t" << result->score[seq_index] << "\t" << result->i[seq_index] <<"\t" << result->j[seq_index] << std::endl;
}
// printf("Template=%-12.12s i=%-4i j=%-4i score=%6.3f\n",t->name,i2,j2,score);
}
/* Routine optimized for shuffling a buffer for a type size larger than 16 bytes. */
static void
shuffle16_tiled_avx2(uint8_t* const dest, const uint8_t* const src,
const size_t vectorizable_elements, const size_t total_elements, const size_t bytesoftype)
{
size_t j;
int k, l;
__m256i ymm0[16], ymm1[16];
const lldiv_t vecs_per_el = lldiv(bytesoftype, sizeof(__m128i));
/* Create the shuffle mask.
NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from
most to least significant (i.e., their order is reversed when compared to
loading the mask from an array). */
const __m256i shmask = _mm256_set_epi8(
0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04,
0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00,
0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04,
0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00);
for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) {
/* Advance the offset into the type by the vector size (in bytes), unless this is
the initial iteration and the type size is not a multiple of the vector size.
In that case, only advance by the number of bytes necessary so that the number
of remaining bytes in the type will be a multiple of the vector size. */
size_t offset_into_type;
for (offset_into_type = 0; offset_into_type < bytesoftype;
offset_into_type += (offset_into_type == 0 && vecs_per_el.rem > 0 ? vecs_per_el.rem : sizeof(__m128i))) {
/* Fetch elements in groups of 512 bytes */
const uint8_t* const src_with_offset = src + offset_into_type;
for (k = 0; k < 16; k++) {
ymm0[k] = _mm256_loadu2_m128i(
(__m128i*)(src_with_offset + (j + (2 * k) + 1) * bytesoftype),
(__m128i*)(src_with_offset + (j + (2 * k)) * bytesoftype));
}
/* Transpose bytes */
for (k = 0, l = 0; k < 8; k++, l +=2) {
ymm1[k*2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l+1]);
ymm1[k*2+1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l+1]);
}
/* Transpose words */
for (k = 0, l = -2; k < 8; k++, l++) {
if ((k%2) == 0) l += 2;
ymm0[k*2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l+2]);
ymm0[k*2+1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l+2]);
}
/* Transpose double words */
for (k = 0, l = -4; k < 8; k++, l++) {
if ((k%4) == 0) l += 4;
ymm1[k*2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l+4]);
ymm1[k*2+1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l+4]);
}
/* Transpose quad words */
for (k = 0; k < 8; k++) {
ymm0[k*2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k+8]);
ymm0[k*2+1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k+8]);
}
for (k = 0; k < 16; k++) {
ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8);
ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask);
}
/* Store the result vectors */
uint8_t* const dest_for_jth_element = dest + j;
for (k = 0; k < 16; k++) {
_mm256_storeu_si256((__m256i*)(dest_for_jth_element + (total_elements * (offset_into_type + k))), ymm0[k]);
}
}
}
}
