static WEBP_INLINE void ITransformOne(const uint8_t* ref, const int16_t* in,
uint8_t* dst) {
v8i16 input0, input1;
v4i32 in0, in1, in2, in3, hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3;
v4i32 res0, res1, res2, res3;
v16i8 dest0, dest1, dest2, dest3;
const v16i8 zero = { 0 };
LD_SH2(in, 8, input0, input1);
UNPCK_SH_SW(input0, in0, in1);
UNPCK_SH_SW(input1, in2, in3);
IDCT_1D_W(in0, in1, in2, in3, hz0, hz1, hz2, hz3);
TRANSPOSE4x4_SW_SW(hz0, hz1, hz2, hz3, hz0, hz1, hz2, hz3);
IDCT_1D_W(hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3);
SRARI_W4_SW(vt0, vt1, vt2, vt3, 3);
TRANSPOSE4x4_SW_SW(vt0, vt1, vt2, vt3, vt0, vt1, vt2, vt3);
LD_SB4(ref, BPS, dest0, dest1, dest2, dest3);
ILVR_B4_SW(zero, dest0, zero, dest1, zero, dest2, zero, dest3,
res0, res1, res2, res3);
ILVR_H4_SW(zero, res0, zero, res1, zero, res2, zero, res3,
res0, res1, res2, res3);
ADD4(res0, vt0, res1, vt1, res2, vt2, res3, vt3, res0, res1, res2, res3);
CLIP_SW4_0_255(res0, res1, res2, res3);
PCKEV_B2_SW(res0, res1, res2, res3, vt0, vt1);
res0 = (v4i32)__msa_pckev_b((v16i8)vt0, (v16i8)vt1);
ST4x4_UB(res0, res0, 3, 2, 1, 0, dst, BPS);
}
static void avc_dequant_4x4_msa( int16_t *p_dct, int32_t pi_dequant_mf[6][16],
int32_t i_qp )
{
const int32_t i_mf = i_qp % 6;
const int32_t q_bits = i_qp / 6 - 4;
v8i16 dct0, dct1;
v4i32 dequant_m_f0, dequant_m_f1, dequant_m_f2, dequant_m_f3;
LD_SH2( p_dct, 8, dct0, dct1 );
LD_SW2( pi_dequant_mf[i_mf], 4, dequant_m_f0, dequant_m_f1 );
LD_SW2( pi_dequant_mf[i_mf] + 8, 4, dequant_m_f2, dequant_m_f3 );
if( q_bits >= 0 )
{
v8i16 dequant_mf_h0, dequant_mf_h1, q_bits_vec;
q_bits_vec = __msa_fill_h( q_bits );
PCKEV_H2_SH( dequant_m_f1, dequant_m_f0, dequant_m_f3, dequant_m_f2,
dequant_mf_h0, dequant_mf_h1 );
dct0 *= dequant_mf_h0;
dct1 *= dequant_mf_h1;
dct0 <<= q_bits_vec;
dct1 <<= q_bits_vec;
ST_SH2( dct0, dct1, p_dct, 8 );
}
else
{
const int32_t q_bits_add = 1 << ( -q_bits - 1 );
v4i32 dct_signed_w0, dct_signed_w1, dct_signed_w2, dct_signed_w3;
v4i32 q_bits_vec, q_bits_vec_add;
q_bits_vec_add = __msa_fill_w( q_bits_add );
q_bits_vec = __msa_fill_w( -q_bits );
UNPCK_SH_SW( dct0, dct_signed_w0, dct_signed_w1 );
UNPCK_SH_SW( dct1, dct_signed_w2, dct_signed_w3 );
dct_signed_w0 *= dequant_m_f0;
dct_signed_w1 *= dequant_m_f1;
dct_signed_w2 *= dequant_m_f2;
dct_signed_w3 *= dequant_m_f3;
dct_signed_w0 += q_bits_vec_add;
dct_signed_w1 += q_bits_vec_add;
dct_signed_w2 += q_bits_vec_add;
dct_signed_w3 += q_bits_vec_add;
SRA_4V( dct_signed_w0, dct_signed_w1, dct_signed_w2, dct_signed_w3,
q_bits_vec );
PCKEV_H2_SH( dct_signed_w1, dct_signed_w0, dct_signed_w3, dct_signed_w2,
dct0, dct1 );
ST_SH2( dct0, dct1, p_dct, 8 );
}
}
static int32_t avc_quant_4x4_msa( int16_t *p_dct, uint16_t *p_mf,
uint16_t *p_bias )
{
int32_t non_zero = 0;
v8i16 dct0, dct1;
v8i16 zero = { 0 };
v8i16 dct0_mask, dct1_mask;
v8i16 dct_h0, dct_h1, mf_h0, mf_h1, bias_h0, bias_h1;
v4i32 dct_signed_w0, dct_signed_w1, dct_signed_w2, dct_signed_w3;
v4i32 dct_w0, dct_w1, dct_w2, dct_w3;
v4i32 mf_vec0, mf_vec1, mf_vec2, mf_vec3;
v4i32 bias0, bias1, bias2, bias3;
LD_SH2( p_dct, 8, dct0, dct1 );
LD_SH2( p_bias, 8, bias_h0, bias_h1 );
LD_SH2( p_mf, 8, mf_h0, mf_h1 );
dct0_mask = __msa_clei_s_h( dct0, 0 );
dct1_mask = __msa_clei_s_h( dct1, 0 );
UNPCK_SH_SW( dct0, dct_signed_w0, dct_signed_w1 );
UNPCK_SH_SW( dct1, dct_signed_w2, dct_signed_w3 );
ILVR_H2_SW( zero, bias_h0, zero, bias_h1, bias0, bias2 );
ILVL_H2_SW( zero, bias_h0, zero, bias_h1, bias1, bias3 );
ILVR_H2_SW( zero, mf_h0, zero, mf_h1, mf_vec0, mf_vec2 );
ILVL_H2_SW( zero, mf_h0, zero, mf_h1, mf_vec1, mf_vec3 );
dct_w1 = __msa_add_a_w( dct_signed_w1, bias1 );
dct_w0 = __msa_add_a_w( dct_signed_w0, bias0 );
dct_w2 = __msa_add_a_w( dct_signed_w2, bias2 );
dct_w3 = __msa_add_a_w( dct_signed_w3, bias3 );
dct_w0 *= mf_vec0;
dct_w1 *= mf_vec1;
dct_w2 *= mf_vec2;
dct_w3 *= mf_vec3;
SRA_4V( dct_w0, dct_w1, dct_w2, dct_w3, 16 );
PCKEV_H2_SH( dct_w1, dct_w0, dct_w3, dct_w2, dct_h0, dct_h1 );
dct0 = zero - dct_h0;
dct1 = zero - dct_h1;
dct0 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h0, ( v16u8 ) dct0,
( v16u8 ) dct0_mask );
dct1 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h1, ( v16u8 ) dct1,
( v16u8 ) dct1_mask );
non_zero = HADD_SW_S32( ( v4u32 ) ( dct_h0 + dct_h1 ) );
ST_SH2( dct0, dct1, p_dct, 8 );
return !!non_zero;
}
static void avc_dequant_4x4_dc_msa( int16_t *p_dct,
int32_t pi_dequant_mf[6][16],
int32_t i_qp )
{
const int32_t q_bits = i_qp / 6 - 6;
int32_t i_dmf = pi_dequant_mf[i_qp % 6][0];
v8i16 dct0, dct1, dequant_mf_h;
LD_SH2( p_dct, 8, dct0, dct1 );
if( q_bits >= 0 )
{
i_dmf <<= q_bits;
dequant_mf_h = __msa_fill_h( i_dmf );
dct0 = dct0 * dequant_mf_h;
dct1 = dct1 * dequant_mf_h;
ST_SH2( dct0, dct1, p_dct, 8 );
}
else
{
const int32_t q_bits_add = 1 << ( -q_bits - 1 );
v4i32 dequant_m_f, q_bits_vec, q_bits_vec_add;
v4i32 dct_signed_w0, dct_signed_w1, dct_signed_w2, dct_signed_w3;
q_bits_vec_add = __msa_fill_w( q_bits_add );
q_bits_vec = __msa_fill_w( -q_bits );
dequant_m_f = __msa_fill_w( i_dmf );
UNPCK_SH_SW( dct0, dct_signed_w0, dct_signed_w1 );
UNPCK_SH_SW( dct1, dct_signed_w2, dct_signed_w3 );
dct_signed_w0 *= dequant_m_f;
dct_signed_w1 *= dequant_m_f;
dct_signed_w2 *= dequant_m_f;
dct_signed_w3 *= dequant_m_f;
dct_signed_w0 += q_bits_vec_add;
dct_signed_w1 += q_bits_vec_add;
dct_signed_w2 += q_bits_vec_add;
dct_signed_w3 += q_bits_vec_add;
SRA_4V( dct_signed_w0, dct_signed_w1, dct_signed_w2, dct_signed_w3,
q_bits_vec );
PCKEV_H2_SH( dct_signed_w1, dct_signed_w0, dct_signed_w3, dct_signed_w2,
dct0, dct1 );
ST_SH2( dct0, dct1, p_dct, 8 );
}
}
static int32_t avc_quant_4x4_dc_msa( int16_t *p_dct, int32_t i_mf,
int32_t i_bias )
{
int32_t non_zero = 0;
v8i16 dct0, dct1, dct0_mask, dct1_mask;
v8i16 zero = { 0 };
v8i16 dct_h0, dct_h1;
v4i32 dct_signed_w0, dct_signed_w1, dct_signed_w2, dct_signed_w3;
v4i32 dct_w0, dct_w1, dct_w2, dct_w3;
v4i32 mf_vec, bias_vec;
LD_SH2( p_dct, 8, dct0, dct1 );
dct0_mask = __msa_clei_s_h( dct0, 0 );
dct1_mask = __msa_clei_s_h( dct1, 0 );
UNPCK_SH_SW( dct0, dct_signed_w0, dct_signed_w1 );
UNPCK_SH_SW( dct1, dct_signed_w2, dct_signed_w3 );
bias_vec = __msa_fill_w( i_bias );
mf_vec = __msa_fill_w( i_mf );
dct_w0 = __msa_add_a_w( dct_signed_w0, bias_vec );
dct_w1 = __msa_add_a_w( dct_signed_w1, bias_vec );
dct_w2 = __msa_add_a_w( dct_signed_w2, bias_vec );
dct_w3 = __msa_add_a_w( dct_signed_w3, bias_vec );
dct_w0 *= mf_vec;
dct_w1 *= mf_vec;
dct_w2 *= mf_vec;
dct_w3 *= mf_vec;
SRA_4V( dct_w0, dct_w1, dct_w2, dct_w3, 16 );
PCKEV_H2_SH( dct_w1, dct_w0, dct_w3, dct_w2, dct_h0, dct_h1 );
dct0 = zero - dct_h0;
dct1 = zero - dct_h1;
dct0 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h0,
( v16u8 ) dct0, ( v16u8 ) dct0_mask );
dct1 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h1,
( v16u8 ) dct1, ( v16u8 ) dct1_mask );
non_zero = HADD_SW_S32( ( v4u32 ) ( dct_h0 + dct_h1 ) );
ST_SH2( dct0, dct1, p_dct, 8 );
return !!non_zero;
}
static void hevc_idct_luma_4x4_msa(int16_t *coeffs)
{
v8i16 in0, in1, dst0, dst1;
v4i32 in_r0, in_l0, in_r1, in_l1, res0, res1, res2, res3;
LD_SH2(coeffs, 8, in0, in1);
UNPCK_SH_SW(in0, in_r0, in_l0);
UNPCK_SH_SW(in1, in_r1, in_l1);
HEVC_IDCT_LUMA4x4_COL(in_r0, in_l0, in_r1, in_l1, res0, res1, res2, res3,
7);
TRANSPOSE4x4_SW_SW(res0, res1, res2, res3, in_r0, in_l0, in_r1, in_l1);
HEVC_IDCT_LUMA4x4_COL(in_r0, in_l0, in_r1, in_l1, res0, res1, res2, res3,
12);
/* Pack and transpose */
PCKEV_H2_SH(res2, res0, res3, res1, dst0, dst1);
ILVRL_H2_SW(dst1, dst0, res0, res1);
ILVRL_W2_SH(res1, res0, dst0, dst1);
ST_SH2(dst0, dst1, coeffs, 8);
}
static void avc_idct8_addblk_msa( uint8_t *p_dst, int16_t *p_src,
int32_t i_dst_stride )
{
v8i16 src0, src1, src2, src3, src4, src5, src6, src7;
v8i16 vec0, vec1, vec2, vec3;
v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
v4i32 tmp0_r, tmp1_r, tmp2_r, tmp3_r, tmp4_r, tmp5_r, tmp6_r, tmp7_r;
v4i32 tmp0_l, tmp1_l, tmp2_l, tmp3_l, tmp4_l, tmp5_l, tmp6_l, tmp7_l;
v4i32 vec0_r, vec1_r, vec2_r, vec3_r, vec0_l, vec1_l, vec2_l, vec3_l;
v4i32 res0_r, res1_r, res2_r, res3_r, res4_r, res5_r, res6_r, res7_r;
v4i32 res0_l, res1_l, res2_l, res3_l, res4_l, res5_l, res6_l, res7_l;
v16i8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
v16i8 zeros = { 0 };
p_src[ 0 ] += 32;
LD_SH8( p_src, 8, src0, src1, src2, src3, src4, src5, src6, src7 );
vec0 = src0 + src4;
vec1 = src0 - src4;
vec2 = src2 >> 1;
vec2 = vec2 - src6;
vec3 = src6 >> 1;
vec3 = src2 + vec3;
BUTTERFLY_4( vec0, vec1, vec2, vec3, tmp0, tmp1, tmp2, tmp3 );
vec0 = src7 >> 1;
vec0 = src5 - vec0 - src3 - src7;
vec1 = src3 >> 1;
vec1 = src1 - vec1 + src7 - src3;
vec2 = src5 >> 1;
vec2 = vec2 - src1 + src7 + src5;
vec3 = src1 >> 1;
vec3 = vec3 + src3 + src5 + src1;
tmp4 = vec3 >> 2;
tmp4 += vec0;
tmp5 = vec2 >> 2;
tmp5 += vec1;
tmp6 = vec1 >> 2;
tmp6 -= vec2;
tmp7 = vec0 >> 2;
tmp7 = vec3 - tmp7;
BUTTERFLY_8( tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7,
res0, res1, res2, res3, res4, res5, res6, res7 );
TRANSPOSE8x8_SH_SH( res0, res1, res2, res3, res4, res5, res6, res7,
res0, res1, res2, res3, res4, res5, res6, res7 );
UNPCK_SH_SW( res0, tmp0_r, tmp0_l );
UNPCK_SH_SW( res1, tmp1_r, tmp1_l );
UNPCK_SH_SW( res2, tmp2_r, tmp2_l );
UNPCK_SH_SW( res3, tmp3_r, tmp3_l );
UNPCK_SH_SW( res4, tmp4_r, tmp4_l );
UNPCK_SH_SW( res5, tmp5_r, tmp5_l );
UNPCK_SH_SW( res6, tmp6_r, tmp6_l );
UNPCK_SH_SW( res7, tmp7_r, tmp7_l );
BUTTERFLY_4( tmp0_r, tmp0_l, tmp4_l, tmp4_r,
vec0_r, vec0_l, vec1_l, vec1_r );
vec2_r = tmp2_r >> 1;
vec2_l = tmp2_l >> 1;
vec2_r -= tmp6_r;
vec2_l -= tmp6_l;
vec3_r = tmp6_r >> 1;
vec3_l = tmp6_l >> 1;
vec3_r += tmp2_r;
vec3_l += tmp2_l;
BUTTERFLY_4( vec0_r, vec1_r, vec2_r, vec3_r,
tmp0_r, tmp2_r, tmp4_r, tmp6_r );
BUTTERFLY_4( vec0_l, vec1_l, vec2_l, vec3_l,
tmp0_l, tmp2_l, tmp4_l, tmp6_l );
vec0_r = tmp7_r >> 1;
vec0_l = tmp7_l >> 1;
vec0_r = tmp5_r - vec0_r - tmp3_r - tmp7_r;
vec0_l = tmp5_l - vec0_l - tmp3_l - tmp7_l;
vec1_r = tmp3_r >> 1;
vec1_l = tmp3_l >> 1;
vec1_r = tmp1_r - vec1_r + tmp7_r - tmp3_r;
vec1_l = tmp1_l - vec1_l + tmp7_l - tmp3_l;
vec2_r = tmp5_r >> 1;
vec2_l = tmp5_l >> 1;
vec2_r = vec2_r - tmp1_r + tmp7_r + tmp5_r;
vec2_l = vec2_l - tmp1_l + tmp7_l + tmp5_l;
vec3_r = tmp1_r >> 1;
vec3_l = tmp1_l >> 1;
vec3_r = vec3_r + tmp3_r + tmp5_r + tmp1_r;
vec3_l = vec3_l + tmp3_l + tmp5_l + tmp1_l;
tmp1_r = vec3_r >> 2;
tmp1_l = vec3_l >> 2;
tmp1_r += vec0_r;
tmp1_l += vec0_l;
tmp3_r = vec2_r >> 2;
tmp3_l = vec2_l >> 2;
tmp3_r += vec1_r;
tmp3_l += vec1_l;
tmp5_r = vec1_r >> 2;
tmp5_l = vec1_l >> 2;
tmp5_r -= vec2_r;
tmp5_l -= vec2_l;
tmp7_r = vec0_r >> 2;
tmp7_l = vec0_l >> 2;
tmp7_r = vec3_r - tmp7_r;
tmp7_l = vec3_l - tmp7_l;
BUTTERFLY_4( tmp0_r, tmp0_l, tmp7_l, tmp7_r,
res0_r, res0_l, res7_l, res7_r );
BUTTERFLY_4( tmp2_r, tmp2_l, tmp5_l, tmp5_r,
res1_r, res1_l, res6_l, res6_r );
BUTTERFLY_4( tmp4_r, tmp4_l, tmp3_l, tmp3_r,
res2_r, res2_l, res5_l, res5_r );
BUTTERFLY_4( tmp6_r, tmp6_l, tmp1_l, tmp1_r,
res3_r, res3_l, res4_l, res4_r );
SRA_4V( res0_r, res0_l, res1_r, res1_l, 6 );
SRA_4V( res2_r, res2_l, res3_r, res3_l, 6 );
SRA_4V( res4_r, res4_l, res5_r, res5_l, 6 );
SRA_4V( res6_r, res6_l, res7_r, res7_l, 6 );
PCKEV_H4_SH( res0_l, res0_r, res1_l, res1_r, res2_l, res2_r, res3_l, res3_r,
res0, res1, res2, res3 );
PCKEV_H4_SH( res4_l, res4_r, res5_l, res5_r, res6_l, res6_r, res7_l, res7_r,
res4, res5, res6, res7 );
LD_SB8( p_dst, i_dst_stride,
dst0, dst1, dst2, dst3,
dst4, dst5, dst6, dst7 );
ILVR_B4_SH( zeros, dst0, zeros, dst1, zeros, dst2, zeros, dst3,
tmp0, tmp1, tmp2, tmp3 );
ILVR_B4_SH( zeros, dst4, zeros, dst5, zeros, dst6, zeros, dst7,
tmp4, tmp5, tmp6, tmp7 );
ADD4( res0, tmp0, res1, tmp1, res2, tmp2, res3, tmp3,
res0, res1, res2, res3 );
ADD4( res4, tmp4, res5, tmp5, res6, tmp6, res7, tmp7,
res4, res5, res6, res7 );
CLIP_SH4_0_255( res0, res1, res2, res3 );
CLIP_SH4_0_255( res4, res5, res6, res7 );
PCKEV_B4_SB( res1, res0, res3, res2, res5, res4, res7, res6,
dst0, dst1, dst2, dst3 );
ST8x4_UB( dst0, dst1, p_dst, i_dst_stride );
p_dst += ( 4 * i_dst_stride );
ST8x4_UB( dst2, dst3, p_dst, i_dst_stride );
}
void
yuv_abgr_convert_msa (JSAMPROW p_in_y, JSAMPROW p_in_cb, JSAMPROW p_in_cr,
JSAMPROW p_rgb, JDIMENSION out_width)
{
int y, cb, cr;
unsigned int col, num_cols_mul_16 = out_width >> 4;
unsigned int remaining_wd = out_width & 0xF;
v16i8 alpha = __msa_ldi_b(0xFF);
v16i8 const_128 = __msa_ldi_b(128);
v16u8 out0, out1, out2, out3, input_y = {0};
v16i8 input_cb, input_cr, out_rgb0, out_rgb1, out_ab0, out_ab1;
v8i16 y_h0, y_h1, cb_h0, cb_h1, cr_h0, cr_h1;
v4i32 cb_w0, cb_w1, cb_w2, cb_w3, cr_w0, cr_w1, cr_w2, cr_w3, zero = {0};
v16i8 out_r0, out_g0, out_b0;
for (col = num_cols_mul_16; col--;) {
input_y = LD_UB(p_in_y);
input_cb = LD_SB(p_in_cb);
input_cr = LD_SB(p_in_cr);
p_in_y += 16;
p_in_cb += 16;
p_in_cr += 16;
input_cb -= const_128;
input_cr -= const_128;
UNPCK_UB_SH(input_y, y_h0, y_h1);
UNPCK_SB_SH(input_cb, cb_h0, cb_h1);
UNPCK_SB_SH(input_cr, cr_h0, cr_h1);
CALC_G4_FRM_YUV(y_h0, y_h1, cb_h0, cb_h1, cr_h0, cr_h1, out_g0);
UNPCK_SH_SW(cr_h0, cr_w0, cr_w1);
UNPCK_SH_SW(cr_h1, cr_w2, cr_w3);
CALC_R4_FRM_YUV(y_h0, y_h1, cr_w0, cr_w1, cr_w2, cr_w3, out_r0);
UNPCK_SH_SW(cb_h0, cb_w0, cb_w1);
UNPCK_SH_SW(cb_h1, cb_w2, cb_w3);
CALC_B4_FRM_YUV(y_h0, y_h1, cb_w0, cb_w1, cb_w2, cb_w3, out_b0);
ILVRL_B2_SB(out_r0, out_g0, out_rgb0, out_rgb1);
ILVRL_B2_SB(out_b0, alpha, out_ab0, out_ab1);
ILVRL_H2_UB(out_rgb0, out_ab0, out0, out1);
ILVRL_H2_UB(out_rgb1, out_ab1, out2, out3);
ST_UB4(out0, out1, out2, out3, p_rgb, 16);
p_rgb += 16 * 4;
}
if (remaining_wd >= 8) {
uint64_t in_y, in_cb, in_cr;
v16i8 input_cbcr = {0};
in_y = LD(p_in_y);
in_cb = LD(p_in_cb);
in_cr = LD(p_in_cr);
p_in_y += 8;
p_in_cb += 8;
p_in_cr += 8;
input_y = (v16u8) __msa_insert_d((v2i64) input_y, 0, in_y);
input_cbcr = (v16i8) __msa_insert_d((v2i64) input_cbcr, 0, in_cb);
input_cbcr = (v16i8) __msa_insert_d((v2i64) input_cbcr, 1, in_cr);
input_cbcr -= const_128;
y_h0 = (v8i16) __msa_ilvr_b((v16i8) zero, (v16i8) input_y);
UNPCK_SB_SH(input_cbcr, cb_h0, cr_h0);
UNPCK_SH_SW(cb_h0, cb_w0, cb_w1);
UNPCK_SH_SW(cr_h0, cr_w0, cr_w1);
CALC_R2_FRM_YUV(y_h0, cr_w0, cr_w1, out_r0);
CALC_G2_FRM_YUV(y_h0, cb_h0, cr_h0, out_g0);
CALC_B2_FRM_YUV(y_h0, cb_w0, cb_w1, out_b0);
out_rgb0 = (v16i8) __msa_ilvr_b((v16i8) out_r0, (v16i8) out_g0);
out_ab0 = (v16i8) __msa_ilvr_b((v16i8) out_b0, alpha);
ILVRL_H2_UB(out_rgb0, out_ab0, out0, out1);
ST_UB2(out0, out1, p_rgb, 16);
p_rgb += 16 * 2;
remaining_wd -= 8;
}
for (col = 0; col < remaining_wd; col++) {
y = (int) (p_in_y[col]);
cb = (int) (p_in_cb[col]) - 128;
cr = (int) (p_in_cr[col]) - 128;
p_rgb[0] = 0xFF;
p_rgb[1] = clip_pixel(y + ROUND_POWER_OF_TWO(FIX_1_77200 * cb, 16));
p_rgb[2] = clip_pixel(y + ROUND_POWER_OF_TWO(((-FIX_0_34414) * cb -
FIX_0_71414 * cr), 16));
p_rgb[3] = clip_pixel(y + ROUND_POWER_OF_TWO(FIX_1_40200 * cr, 16));
p_rgb += 4;
}
}
void
yuv_bgr_convert_msa (JSAMPROW p_in_y, JSAMPROW p_in_cb, JSAMPROW p_in_cr,
JSAMPROW p_rgb, JDIMENSION out_width)
{
int32_t y, cb, cr;
uint32_t col, num_cols_mul_16 = out_width >> 4;
uint32_t remaining_wd = out_width & 0xF;
v16u8 mask_rgb0 = {0, 1, 16, 2, 3, 17, 4, 5, 18, 6, 7, 19, 8, 9, 20, 10};
v16u8 mask_rgb1 = {11, 21, 12, 13, 22, 14, 15, 23, 0, 1, 24, 2, 3, 25, 4, 5};
v16u8 mask_rgb2 = {26, 6, 7, 27, 8, 9, 28, 10, 11, 29, 12, 13, 30, 14, 15, 31};
v16u8 tmp0, tmp1, out0, out1, out2, input_y = {0};
v16i8 input_cb, input_cr, out_rgb0, out_rgb1, const_128 = __msa_ldi_b(128);
v8i16 y_h0, y_h1, cb_h0, cb_h1, cr_h0, cr_h1;
v4i32 cb_w0, cb_w1, cb_w2, cb_w3, cr_w0, cr_w1, cr_w2, cr_w3, zero = {0};
v16i8 out_r0, out_g0, out_b0;
for (col = num_cols_mul_16; col--;) {
input_y = LD_UB(p_in_y);
input_cb = LD_SB(p_in_cb);
input_cr = LD_SB(p_in_cr);
p_in_y += 16;
p_in_cb += 16;
p_in_cr += 16;
input_cb -= const_128;
input_cr -= const_128;
UNPCK_UB_SH(input_y, y_h0, y_h1);
UNPCK_SB_SH(input_cb, cb_h0, cb_h1);
UNPCK_SB_SH(input_cr, cr_h0, cr_h1);
CALC_G4_FRM_YUV(y_h0, y_h1, cb_h0, cb_h1, cr_h0, cr_h1, out_g0);
UNPCK_SH_SW(cr_h0, cr_w0, cr_w1);
UNPCK_SH_SW(cr_h1, cr_w2, cr_w3);
CALC_R4_FRM_YUV(y_h0, y_h1, cr_w0, cr_w1, cr_w2, cr_w3, out_r0);
UNPCK_SH_SW(cb_h0, cb_w0, cb_w1);
UNPCK_SH_SW(cb_h1, cb_w2, cb_w3);
CALC_B4_FRM_YUV(y_h0, y_h1, cb_w0, cb_w1, cb_w2, cb_w3, out_b0);
ILVRL_B2_SB(out_g0, out_b0, out_rgb0, out_rgb1);
VSHF_B2_UB(out_rgb0, out_r0, out_rgb0, out_r0, mask_rgb0, mask_rgb1,
out0, tmp0);
VSHF_B2_UB(out_rgb1, out_r0, out_rgb1, out_r0, mask_rgb1, mask_rgb2,
tmp1, out2);
out1 = (v16u8) __msa_sldi_b((v16i8) zero, (v16i8) tmp1, 8);
out1 = (v16u8) __msa_pckev_d((v2i64) out1, (v2i64) tmp0);
ST_UB(out0, p_rgb);
p_rgb += 16;
ST_UB(out1, p_rgb);
p_rgb += 16;
ST_UB(out2, p_rgb);
p_rgb += 16;
}
if (remaining_wd >= 8) {
uint64_t in_y, in_cb, in_cr;
v16i8 input_cbcr = {0};
in_y = LD(p_in_y);
in_cb = LD(p_in_cb);
in_cr = LD(p_in_cr);
p_in_y += 8;
p_in_cb += 8;
p_in_cr += 8;
input_y = (v16u8) __msa_insert_d((v2i64) input_y, 0, in_y);
input_cbcr = (v16i8) __msa_insert_d((v2i64) input_cbcr, 0, in_cb);
input_cbcr = (v16i8) __msa_insert_d((v2i64) input_cbcr, 1, in_cr);
input_cbcr -= const_128;
y_h0 = (v8i16) __msa_ilvr_b((v16i8) zero, (v16i8) input_y);
UNPCK_SB_SH(input_cbcr, cb_h0, cr_h0);
UNPCK_SH_SW(cb_h0, cb_w0, cb_w1);
UNPCK_SH_SW(cr_h0, cr_w0, cr_w1);
CALC_R2_FRM_YUV(y_h0, cr_w0, cr_w1, out_r0);
CALC_G2_FRM_YUV(y_h0, cb_h0, cr_h0, out_g0);
CALC_B2_FRM_YUV(y_h0, cb_w0, cb_w1, out_b0);
out_rgb0 = (v16i8) __msa_ilvr_b((v16i8) out_g0, (v16i8) out_b0);
VSHF_B2_UB(out_rgb0, out_r0, out_rgb0, out_r0, mask_rgb0, mask_rgb1,
out0, out1);
ST_UB(out0, p_rgb);
p_rgb += 16;
ST8x1_UB(out1, p_rgb);
p_rgb += 8;
remaining_wd -= 8;
}
for (col = 0; col < remaining_wd; col++) {
y = (int) (p_in_y[col]);
cb = (int) (p_in_cb[col]) - 128;
cr = (int) (p_in_cr[col]) - 128;
/* Range-limiting is essential due to noise introduced by DCT losses. */
p_rgb[0] = clip_pixel(y + ROUND_POWER_OF_TWO(FIX_1_77200 * cb, 16));
p_rgb[1] = clip_pixel(y + ROUND_POWER_OF_TWO(((-FIX_0_34414) * cb -
FIX_0_71414 * cr), 16));
p_rgb[2] = clip_pixel(y + ROUND_POWER_OF_TWO(FIX_1_40200 * cr, 16));
p_rgb += 3;
}
}
static void temporal_filter_apply_16size_msa(uint8_t *frame1_ptr,
uint32_t stride,
uint8_t *frame2_ptr,
int32_t strength_in,
int32_t filter_wt_in,
uint32_t *acc, uint16_t *cnt)
{
uint32_t row;
v16i8 frame1_0_b, frame1_1_b, frame2_0_b, frame2_1_b;
v16u8 frame_l, frame_h;
v16i8 zero = { 0 };
v8i16 frame2_0_h, frame2_1_h, mod0_h, mod1_h;
v8i16 diff0, diff1, cnt0, cnt1;
v4i32 const3, const16, filter_wt, strength;
v4i32 mod0_w, mod1_w, mod2_w, mod3_w;
v4i32 diff0_r, diff0_l, diff1_r, diff1_l;
v4i32 frame2_0, frame2_1, frame2_2, frame2_3;
v4i32 acc0, acc1, acc2, acc3;
filter_wt = __msa_fill_w(filter_wt_in);
strength = __msa_fill_w(strength_in);
const3 = __msa_ldi_w(3);
const16 = __msa_ldi_w(16);
for (row = 8; row--;)
{
frame1_0_b = LD_SB(frame1_ptr);
frame2_0_b = LD_SB(frame2_ptr);
frame1_ptr += stride;
frame2_ptr += 16;
frame1_1_b = LD_SB(frame1_ptr);
frame2_1_b = LD_SB(frame2_ptr);
LD_SW2(acc, 4, acc0, acc1);
LD_SW2(acc + 8, 4, acc2, acc3);
LD_SH2(cnt, 8, cnt0, cnt1);
ILVRL_B2_UB(frame1_0_b, frame2_0_b, frame_l, frame_h);
HSUB_UB2_SH(frame_l, frame_h, diff0, diff1);
UNPCK_SH_SW(diff0, diff0_r, diff0_l);
UNPCK_SH_SW(diff1, diff1_r, diff1_l);
MUL4(diff0_r, diff0_r, diff0_l, diff0_l, diff1_r, diff1_r, diff1_l,
diff1_l, mod0_w, mod1_w, mod2_w, mod3_w);
MUL4(mod0_w, const3, mod1_w, const3, mod2_w, const3, mod3_w, const3,
mod0_w, mod1_w, mod2_w, mod3_w);
SRAR_W4_SW(mod0_w, mod1_w, mod2_w, mod3_w, strength);
diff0_r = (mod0_w < const16);
diff0_l = (mod1_w < const16);
diff1_r = (mod2_w < const16);
diff1_l = (mod3_w < const16);
SUB4(const16, mod0_w, const16, mod1_w, const16, mod2_w, const16, mod3_w,
mod0_w, mod1_w, mod2_w, mod3_w);
mod0_w = diff0_r & mod0_w;
mod1_w = diff0_l & mod1_w;
mod2_w = diff1_r & mod2_w;
mod3_w = diff1_l & mod3_w;
MUL4(mod0_w, filter_wt, mod1_w, filter_wt, mod2_w, filter_wt, mod3_w,
filter_wt, mod0_w, mod1_w, mod2_w, mod3_w);
PCKEV_H2_SH(mod1_w, mod0_w, mod3_w, mod2_w, mod0_h, mod1_h)
ADD2(mod0_h, cnt0, mod1_h, cnt1, mod0_h, mod1_h);
ST_SH2(mod0_h, mod1_h, cnt, 8);
cnt += 16;
ILVRL_B2_SH(zero, frame2_0_b, frame2_0_h, frame2_1_h);
UNPCK_SH_SW(frame2_0_h, frame2_0, frame2_1);
UNPCK_SH_SW(frame2_1_h, frame2_2, frame2_3);
MUL4(mod0_w, frame2_0, mod1_w, frame2_1, mod2_w, frame2_2, mod3_w,
frame2_3, mod0_w, mod1_w, mod2_w, mod3_w);
ADD4(mod0_w, acc0, mod1_w, acc1, mod2_w, acc2, mod3_w, acc3,
mod0_w, mod1_w, mod2_w, mod3_w);
ST_SW2(mod0_w, mod1_w, acc, 4);
ST_SW2(mod2_w, mod3_w, acc + 8, 4);
acc += 16;
LD_SW2(acc, 4, acc0, acc1);
LD_SW2(acc + 8, 4, acc2, acc3);
LD_SH2(cnt, 8, cnt0, cnt1);
ILVRL_B2_UB(frame1_1_b, frame2_1_b, frame_l, frame_h);
HSUB_UB2_SH(frame_l, frame_h, diff0, diff1);
UNPCK_SH_SW(diff0, diff0_r, diff0_l);
UNPCK_SH_SW(diff1, diff1_r, diff1_l);
MUL4(diff0_r, diff0_r, diff0_l, diff0_l, diff1_r, diff1_r, diff1_l,
diff1_l, mod0_w, mod1_w, mod2_w, mod3_w);
MUL4(mod0_w, const3, mod1_w, const3, mod2_w, const3, mod3_w, const3,
mod0_w, mod1_w, mod2_w, mod3_w);
SRAR_W4_SW(mod0_w, mod1_w, mod2_w, mod3_w, strength);
diff0_r = (mod0_w < const16);
diff0_l = (mod1_w < const16);
diff1_r = (mod2_w < const16);
diff1_l = (mod3_w < const16);
SUB4(const16, mod0_w, const16, mod1_w, const16, mod2_w, const16, mod3_w,
mod0_w, mod1_w, mod2_w, mod3_w);
mod0_w = diff0_r & mod0_w;
mod1_w = diff0_l & mod1_w;
mod2_w = diff1_r & mod2_w;
mod3_w = diff1_l & mod3_w;
MUL4(mod0_w, filter_wt, mod1_w, filter_wt, mod2_w, filter_wt, mod3_w,
filter_wt, mod0_w, mod1_w, mod2_w, mod3_w);
PCKEV_H2_SH(mod1_w, mod0_w, mod3_w, mod2_w, mod0_h, mod1_h);
ADD2(mod0_h, cnt0, mod1_h, cnt1, mod0_h, mod1_h);
ST_SH2(mod0_h, mod1_h, cnt, 8);
cnt += 16;
UNPCK_UB_SH(frame2_1_b, frame2_0_h, frame2_1_h);
UNPCK_SH_SW(frame2_0_h, frame2_0, frame2_1);
UNPCK_SH_SW(frame2_1_h, frame2_2, frame2_3);
MUL4(mod0_w, frame2_0, mod1_w, frame2_1, mod2_w, frame2_2, mod3_w,
frame2_3, mod0_w, mod1_w, mod2_w, mod3_w);
ADD4(mod0_w, acc0, mod1_w, acc1, mod2_w, acc2, mod3_w, acc3,
mod0_w, mod1_w, mod2_w, mod3_w);
ST_SW2(mod0_w, mod1_w, acc, 4);
ST_SW2(mod2_w, mod3_w, acc + 8, 4);
acc += 16;
frame1_ptr += stride;
frame2_ptr += 16;
}
}
static void temporal_filter_apply_8size_msa(uint8_t *frame1_ptr,
uint32_t stride,
uint8_t *frame2_ptr,
int32_t strength_in,
int32_t filter_wt_in,
uint32_t *acc, uint16_t *cnt)
{
uint32_t row;
uint64_t f0, f1, f2, f3, f4, f5, f6, f7;
v16i8 frame1 = { 0 };
v16i8 frame2 = { 0 };
v16i8 frame3 = { 0 };
v16i8 frame4 = { 0 };
v16u8 frame_l, frame_h;
v8i16 frame2_0_h, frame2_1_h, mod0_h, mod1_h;
v8i16 diff0, diff1, cnt0, cnt1;
v4i32 const3, const16;
v4i32 filter_wt, strength;
v4i32 mod0_w, mod1_w, mod2_w, mod3_w;
v4i32 diff0_r, diff0_l, diff1_r, diff1_l;
v4i32 frame2_0, frame2_1, frame2_2, frame2_3;
v4i32 acc0, acc1, acc2, acc3;
filter_wt = __msa_fill_w(filter_wt_in);
strength = __msa_fill_w(strength_in);
const3 = __msa_ldi_w(3);
const16 = __msa_ldi_w(16);
for (row = 2; row--;)
{
LD2(frame1_ptr, stride, f0, f1);
frame1_ptr += (2 * stride);
LD2(frame2_ptr, 8, f2, f3);
frame2_ptr += 16;
LD2(frame1_ptr, stride, f4, f5);
frame1_ptr += (2 * stride);
LD2(frame2_ptr, 8, f6, f7);
frame2_ptr += 16;
LD_SW2(acc, 4, acc0, acc1);
LD_SW2(acc + 8, 4, acc2, acc3);
LD_SH2(cnt, 8, cnt0, cnt1);
INSERT_D2_SB(f0, f1, frame1);
INSERT_D2_SB(f2, f3, frame2);
INSERT_D2_SB(f4, f5, frame3);
INSERT_D2_SB(f6, f7, frame4);
ILVRL_B2_UB(frame1, frame2, frame_l, frame_h);
HSUB_UB2_SH(frame_l, frame_h, diff0, diff1);
UNPCK_SH_SW(diff0, diff0_r, diff0_l);
UNPCK_SH_SW(diff1, diff1_r, diff1_l);
MUL4(diff0_r, diff0_r, diff0_l, diff0_l, diff1_r, diff1_r, diff1_l,
diff1_l, mod0_w, mod1_w, mod2_w, mod3_w);
MUL4(mod0_w, const3, mod1_w, const3, mod2_w, const3, mod3_w, const3,
mod0_w, mod1_w, mod2_w, mod3_w);
SRAR_W4_SW(mod0_w, mod1_w, mod2_w, mod3_w, strength);
diff0_r = (mod0_w < const16);
diff0_l = (mod1_w < const16);
diff1_r = (mod2_w < const16);
diff1_l = (mod3_w < const16);
SUB4(const16, mod0_w, const16, mod1_w, const16, mod2_w, const16, mod3_w,
mod0_w, mod1_w, mod2_w, mod3_w);
mod0_w = diff0_r & mod0_w;
mod1_w = diff0_l & mod1_w;
mod2_w = diff1_r & mod2_w;
mod3_w = diff1_l & mod3_w;
MUL4(mod0_w, filter_wt, mod1_w, filter_wt, mod2_w, filter_wt, mod3_w,
filter_wt, mod0_w, mod1_w, mod2_w, mod3_w);
PCKEV_H2_SH(mod1_w, mod0_w, mod3_w, mod2_w, mod0_h, mod1_h);
ADD2(mod0_h, cnt0, mod1_h, cnt1, mod0_h, mod1_h);
ST_SH2(mod0_h, mod1_h, cnt, 8);
cnt += 16;
UNPCK_UB_SH(frame2, frame2_0_h, frame2_1_h);
UNPCK_SH_SW(frame2_0_h, frame2_0, frame2_1);
UNPCK_SH_SW(frame2_1_h, frame2_2, frame2_3);
MUL4(mod0_w, frame2_0, mod1_w, frame2_1, mod2_w, frame2_2, mod3_w,
frame2_3, mod0_w, mod1_w, mod2_w, mod3_w);
ADD4(mod0_w, acc0, mod1_w, acc1, mod2_w, acc2, mod3_w, acc3,
mod0_w, mod1_w, mod2_w, mod3_w);
ST_SW2(mod0_w, mod1_w, acc, 4);
ST_SW2(mod2_w, mod3_w, acc + 8, 4);
acc += 16;
LD_SW2(acc, 4, acc0, acc1);
LD_SW2(acc + 8, 4, acc2, acc3);
LD_SH2(cnt, 8, cnt0, cnt1);
ILVRL_B2_UB(frame3, frame4, frame_l, frame_h);
HSUB_UB2_SH(frame_l, frame_h, diff0, diff1);
UNPCK_SH_SW(diff0, diff0_r, diff0_l);
UNPCK_SH_SW(diff1, diff1_r, diff1_l);
MUL4(diff0_r, diff0_r, diff0_l, diff0_l, diff1_r, diff1_r, diff1_l,
diff1_l, mod0_w, mod1_w, mod2_w, mod3_w);
MUL4(mod0_w, const3, mod1_w, const3, mod2_w, const3, mod3_w, const3,
mod0_w, mod1_w, mod2_w, mod3_w);
SRAR_W4_SW(mod0_w, mod1_w, mod2_w, mod3_w, strength);
diff0_r = (mod0_w < const16);
diff0_l = (mod1_w < const16);
diff1_r = (mod2_w < const16);
diff1_l = (mod3_w < const16);
SUB4(const16, mod0_w, const16, mod1_w, const16, mod2_w, const16, mod3_w,
mod0_w, mod1_w, mod2_w, mod3_w);
mod0_w = diff0_r & mod0_w;
mod1_w = diff0_l & mod1_w;
mod2_w = diff1_r & mod2_w;
mod3_w = diff1_l & mod3_w;
MUL4(mod0_w, filter_wt, mod1_w, filter_wt, mod2_w, filter_wt, mod3_w,
filter_wt, mod0_w, mod1_w, mod2_w, mod3_w);
PCKEV_H2_SH(mod1_w, mod0_w, mod3_w, mod2_w, mod0_h, mod1_h);
ADD2(mod0_h, cnt0, mod1_h, cnt1, mod0_h, mod1_h);
ST_SH2(mod0_h, mod1_h, cnt, 8);
cnt += 16;
UNPCK_UB_SH(frame4, frame2_0_h, frame2_1_h);
UNPCK_SH_SW(frame2_0_h, frame2_0, frame2_1);
UNPCK_SH_SW(frame2_1_h, frame2_2, frame2_3);
MUL4(mod0_w, frame2_0, mod1_w, frame2_1, mod2_w, frame2_2, mod3_w,
frame2_3, mod0_w, mod1_w, mod2_w, mod3_w);
ADD4(mod0_w, acc0, mod1_w, acc1, mod2_w, acc2, mod3_w, acc3,
mod0_w, mod1_w, mod2_w, mod3_w);
ST_SW2(mod0_w, mod1_w, acc, 4);
ST_SW2(mod2_w, mod3_w, acc + 8, 4);
acc += 16;
}
}
static void fdct8x32_1d_row_even_4x(int16_t *input, int16_t *interm_ptr,
int16_t *out) {
v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
v8i16 in8, in9, in10, in11, in12, in13, in14, in15;
v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
v4i32 vec0_l, vec1_l, vec2_l, vec3_l, vec4_l, vec5_l, vec6_l, vec7_l;
v4i32 vec0_r, vec1_r, vec2_r, vec3_r, vec4_r, vec5_r, vec6_r, vec7_r;
v4i32 tmp0_w, tmp1_w, tmp2_w, tmp3_w;
/* fdct32 even */
/* stage 2 */
LD_SH8(input, 8, in0, in1, in2, in3, in4, in5, in6, in7);
LD_SH8(input + 64, 8, in8, in9, in10, in11, in12, in13, in14, in15);
BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7,
in8, in9, in10, in11, in12, in13, in14, in15,
vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7,
in8, in9, in10, in11, in12, in13, in14, in15);
ST_SH8(vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, interm_ptr, 8);
ST_SH8(in8, in9, in10, in11, in12, in13, in14, in15, interm_ptr + 64, 8);
/* Stage 3 */
UNPCK_SH_SW(vec0, vec0_l, vec0_r);
UNPCK_SH_SW(vec1, vec1_l, vec1_r);
UNPCK_SH_SW(vec2, vec2_l, vec2_r);
UNPCK_SH_SW(vec3, vec3_l, vec3_r);
UNPCK_SH_SW(vec4, vec4_l, vec4_r);
UNPCK_SH_SW(vec5, vec5_l, vec5_r);
UNPCK_SH_SW(vec6, vec6_l, vec6_r);
UNPCK_SH_SW(vec7, vec7_l, vec7_r);
ADD4(vec0_r, vec7_r, vec1_r, vec6_r, vec2_r, vec5_r, vec3_r, vec4_r,
tmp0_w, tmp1_w, tmp2_w, tmp3_w);
BUTTERFLY_4(tmp0_w, tmp1_w, tmp2_w, tmp3_w, vec4_r, vec6_r, vec7_r, vec5_r);
ADD4(vec0_l, vec7_l, vec1_l, vec6_l, vec2_l, vec5_l, vec3_l, vec4_l,
vec0_r, vec1_r, vec2_r, vec3_r);
tmp3_w = vec0_r + vec3_r;
vec0_r = vec0_r - vec3_r;
vec3_r = vec1_r + vec2_r;
vec1_r = vec1_r - vec2_r;
DOTP_CONST_PAIR_W(vec4_r, vec6_r, tmp3_w, vec3_r, cospi_16_64,
cospi_16_64, vec4_r, tmp3_w, vec6_r, vec3_r);
FDCT32_POSTPROC_NEG_W(vec4_r);
FDCT32_POSTPROC_NEG_W(tmp3_w);
FDCT32_POSTPROC_NEG_W(vec6_r);
FDCT32_POSTPROC_NEG_W(vec3_r);
PCKEV_H2_SH(vec4_r, tmp3_w, vec6_r, vec3_r, vec4, vec5);
ST_SH2(vec5, vec4, out, 8);
DOTP_CONST_PAIR_W(vec5_r, vec7_r, vec0_r, vec1_r, cospi_24_64,
cospi_8_64, vec4_r, tmp3_w, vec6_r, vec3_r);
FDCT32_POSTPROC_NEG_W(vec4_r);
FDCT32_POSTPROC_NEG_W(tmp3_w);
FDCT32_POSTPROC_NEG_W(vec6_r);
FDCT32_POSTPROC_NEG_W(vec3_r);
PCKEV_H2_SH(vec4_r, tmp3_w, vec6_r, vec3_r, vec4, vec5);
ST_SH2(vec5, vec4, out + 16, 8);
LD_SH8(interm_ptr, 8, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7);
SUB4(vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, vec5, vec6, vec7);
DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6);
ADD2(vec4, vec5, vec7, vec6, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, in5, in4);
FDCT_POSTPROC_2V_NEG_H(in4, in5);
ST_SH(in4, out + 32);
ST_SH(in5, out + 56);
SUB2(vec4, vec5, vec7, vec6, vec4, vec7);
DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, in5, in4);
FDCT_POSTPROC_2V_NEG_H(in4, in5);
ST_SH(in4, out + 40);
ST_SH(in5, out + 48);
LD_SH8(interm_ptr + 64, 8, in8, in9, in10, in11, in12, in13, in14, in15);
DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5);
DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4);
ADD4(in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, vec1, vec6, in2);
DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3);
ADD2(in0, in1, in2, in3, vec0, vec7);
DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, in5, in4);
FDCT_POSTPROC_2V_NEG_H(in4, in5);
ST_SH(in4, out + 64);
ST_SH(in5, out + 120);
SUB2(in0, in1, in2, in3, in0, in2);
DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, in5, in4);
FDCT_POSTPROC_2V_NEG_H(in4, in5);
ST_SH(in4, out + 72);
ST_SH(in5, out + 112);
SUB2(in9, vec2, in14, vec5, vec2, vec5);
DOTP_CONST_PAIR((-vec2), vec5, cospi_24_64, cospi_8_64, in2, in1);
SUB4(in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, vec2, vec5);
DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, in5, in4);
FDCT_POSTPROC_2V_NEG_H(in4, in5);
ST_SH(in4, out + 80);
ST_SH(in5, out + 104);
ADD2(in3, in2, in0, in1, vec3, vec4);
DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, in4, in5);
FDCT_POSTPROC_2V_NEG_H(in4, in5);
ST_SH(in4, out + 96);
ST_SH(in5, out + 88);
}
static void avc_dequant_8x8_msa( int16_t *p_dct, int32_t pi_dequant_mf[6][64],
int32_t i_qp )
{
const int32_t i_mf = i_qp % 6;
const int32_t q_bits = i_qp / 6 - 6;
v8i16 dct0, dct1, dct2, dct3, dct4, dct5, dct6, dct7;
v4i32 dequant_m_f0, dequant_m_f1, dequant_m_f2, dequant_m_f3;
v4i32 dequant_m_f4, dequant_m_f5, dequant_m_f6, dequant_m_f7;
v4i32 dequant_m_f8, dequant_m_f9, dequant_m_f10, dequant_m_f11;
v4i32 dequant_m_f12, dequant_m_f13, dequant_m_f14, dequant_m_f15;
LD_SH8( p_dct, 8, dct0, dct1, dct2, dct3, dct4, dct5, dct6, dct7 );
LD_SW2( pi_dequant_mf[i_mf], 4, dequant_m_f0, dequant_m_f1 );
LD_SW2( pi_dequant_mf[i_mf] + 8, 4, dequant_m_f2, dequant_m_f3 );
LD_SW2( pi_dequant_mf[i_mf] + 16, 4, dequant_m_f4, dequant_m_f5 );
LD_SW2( pi_dequant_mf[i_mf] + 24, 4, dequant_m_f6, dequant_m_f7 );
LD_SW2( pi_dequant_mf[i_mf] + 32, 4, dequant_m_f8, dequant_m_f9 );
LD_SW2( pi_dequant_mf[i_mf] + 40, 4, dequant_m_f10, dequant_m_f11 );
LD_SW2( pi_dequant_mf[i_mf] + 48, 4, dequant_m_f12, dequant_m_f13 );
LD_SW2( pi_dequant_mf[i_mf] + 56, 4, dequant_m_f14, dequant_m_f15 );
if( q_bits >= 0 )
{
v8i16 q_bits_vec;
v8i16 dequant_mf_h0, dequant_mf_h1, dequant_mf_h2, dequant_mf_h3;
v8i16 dequant_mf_h4, dequant_mf_h5, dequant_mf_h6, dequant_mf_h7;
q_bits_vec = __msa_fill_h( q_bits );
PCKEV_H4_SH( dequant_m_f1, dequant_m_f0, dequant_m_f3, dequant_m_f2,
dequant_m_f5, dequant_m_f4, dequant_m_f7, dequant_m_f6,
dequant_mf_h0, dequant_mf_h1,
dequant_mf_h2, dequant_mf_h3 );
PCKEV_H4_SH( dequant_m_f9, dequant_m_f8, dequant_m_f11, dequant_m_f10,
dequant_m_f13, dequant_m_f12, dequant_m_f15, dequant_m_f14,
dequant_mf_h4, dequant_mf_h5,
dequant_mf_h6, dequant_mf_h7 );
dct0 *= dequant_mf_h0;
dct1 *= dequant_mf_h1;
dct2 *= dequant_mf_h2;
dct3 *= dequant_mf_h3;
dct4 *= dequant_mf_h4;
dct5 *= dequant_mf_h5;
dct6 *= dequant_mf_h6;
dct7 *= dequant_mf_h7;
SLLI_4V( dct0, dct1, dct2, dct3, q_bits_vec );
SLLI_4V( dct4, dct5, dct6, dct7, q_bits_vec );
ST_SH8( dct0, dct1, dct2, dct3, dct4, dct5, dct6, dct7, p_dct, 8 );
}
else
{
const int32_t q_bits_add = 1 << ( -q_bits - 1 );
v4i32 dct_signed_w0, dct_signed_w1, dct_signed_w2, dct_signed_w3;
v4i32 dct_signed_w4, dct_signed_w5, dct_signed_w6, dct_signed_w7;
v4i32 dct_signed_w8, dct_signed_w9, dct_signed_w10, dct_signed_w11;
v4i32 dct_signed_w12, dct_signed_w13, dct_signed_w14, dct_signed_w15;
v4i32 q_bits_vec, q_bits_vec_add;
q_bits_vec_add = __msa_fill_w( q_bits_add );
q_bits_vec = __msa_fill_w( -q_bits );
UNPCK_SH_SW( dct0, dct_signed_w0, dct_signed_w1 );
UNPCK_SH_SW( dct1, dct_signed_w2, dct_signed_w3 );
UNPCK_SH_SW( dct2, dct_signed_w4, dct_signed_w5 );
UNPCK_SH_SW( dct3, dct_signed_w6, dct_signed_w7 );
UNPCK_SH_SW( dct4, dct_signed_w8, dct_signed_w9 );
UNPCK_SH_SW( dct5, dct_signed_w10, dct_signed_w11 );
UNPCK_SH_SW( dct6, dct_signed_w12, dct_signed_w13 );
UNPCK_SH_SW( dct7, dct_signed_w14, dct_signed_w15 );
dct_signed_w0 *= dequant_m_f0;
dct_signed_w1 *= dequant_m_f1;
dct_signed_w2 *= dequant_m_f2;
dct_signed_w3 *= dequant_m_f3;
dct_signed_w4 *= dequant_m_f4;
dct_signed_w5 *= dequant_m_f5;
dct_signed_w6 *= dequant_m_f6;
dct_signed_w7 *= dequant_m_f7;
dct_signed_w8 *= dequant_m_f8;
dct_signed_w9 *= dequant_m_f9;
dct_signed_w10 *= dequant_m_f10;
dct_signed_w11 *= dequant_m_f11;
dct_signed_w12 *= dequant_m_f12;
dct_signed_w13 *= dequant_m_f13;
dct_signed_w14 *= dequant_m_f14;
dct_signed_w15 *= dequant_m_f15;
dct_signed_w0 += q_bits_vec_add;
dct_signed_w1 += q_bits_vec_add;
dct_signed_w2 += q_bits_vec_add;
dct_signed_w3 += q_bits_vec_add;
dct_signed_w4 += q_bits_vec_add;
dct_signed_w5 += q_bits_vec_add;
dct_signed_w6 += q_bits_vec_add;
dct_signed_w7 += q_bits_vec_add;
dct_signed_w8 += q_bits_vec_add;
dct_signed_w9 += q_bits_vec_add;
dct_signed_w10 += q_bits_vec_add;
dct_signed_w11 += q_bits_vec_add;
dct_signed_w12 += q_bits_vec_add;
dct_signed_w13 += q_bits_vec_add;
dct_signed_w14 += q_bits_vec_add;
dct_signed_w15 += q_bits_vec_add;
SRA_4V( dct_signed_w0, dct_signed_w1, dct_signed_w2, dct_signed_w3,
q_bits_vec );
SRA_4V( dct_signed_w4, dct_signed_w5, dct_signed_w6, dct_signed_w7,
q_bits_vec );
SRA_4V( dct_signed_w8, dct_signed_w9, dct_signed_w10, dct_signed_w11,
q_bits_vec );
SRA_4V( dct_signed_w12, dct_signed_w13, dct_signed_w14, dct_signed_w15,
q_bits_vec );
PCKEV_H4_SH( dct_signed_w1, dct_signed_w0, dct_signed_w3, dct_signed_w2,
dct_signed_w5, dct_signed_w4, dct_signed_w7, dct_signed_w6,
dct0, dct1, dct2, dct3 );
PCKEV_H4_SH( dct_signed_w9, dct_signed_w8, dct_signed_w11,
dct_signed_w10, dct_signed_w13, dct_signed_w12,
dct_signed_w15, dct_signed_w14, dct4, dct5, dct6, dct7 );
ST_SH8( dct0, dct1, dct2, dct3, dct4, dct5, dct6, dct7, p_dct, 8 );
}
}
static int32_t avc_quant_8x8_msa( int16_t *p_dct, uint16_t *p_mf,
uint16_t *p_bias )
{
int32_t non_zero = 0;
v8i16 dct0, dct1, dct2, dct3;
v8i16 zero = { 0 };
v8i16 dct0_mask, dct1_mask, dct2_mask, dct3_mask;
v8i16 dct_h0, dct_h1, dct_h2, dct_h3, mf_h0, mf_h1, mf_h2, mf_h3;
v8i16 bias_h0, bias_h1, bias_h2, bias_h3;
v4i32 dct_w0, dct_w1, dct_w2, dct_w3, dct_w4, dct_w5, dct_w6, dct_w7;
v4i32 dct_signed_w0, dct_signed_w1, dct_signed_w2, dct_signed_w3;
v4i32 dct_signed_w4, dct_signed_w5, dct_signed_w6, dct_signed_w7;
v4i32 mf_vec0, mf_vec1, mf_vec2, mf_vec3;
v4i32 mf_vec4, mf_vec5, mf_vec6, mf_vec7;
v4i32 bias0, bias1, bias2, bias3, bias4, bias5, bias6, bias7;
LD_SH4( p_dct, 8, dct0, dct1, dct2, dct3 );
dct0_mask = __msa_clei_s_h( dct0, 0 );
dct1_mask = __msa_clei_s_h( dct1, 0 );
dct2_mask = __msa_clei_s_h( dct2, 0 );
dct3_mask = __msa_clei_s_h( dct3, 0 );
UNPCK_SH_SW( dct0, dct_signed_w0, dct_signed_w1 );
UNPCK_SH_SW( dct1, dct_signed_w2, dct_signed_w3 );
UNPCK_SH_SW( dct2, dct_signed_w4, dct_signed_w5 );
UNPCK_SH_SW( dct3, dct_signed_w6, dct_signed_w7 );
LD_SH4( p_bias, 8, bias_h0, bias_h1, bias_h2, bias_h3 );
ILVR_H4_SW( zero, bias_h0, zero, bias_h1, zero, bias_h2, zero, bias_h3,
bias0, bias2, bias4, bias6 );
ILVL_H4_SW( zero, bias_h0, zero, bias_h1, zero, bias_h2, zero, bias_h3,
bias1, bias3, bias5, bias7 );
LD_SH4( p_mf, 8, mf_h0, mf_h1, mf_h2, mf_h3 );
ILVR_H4_SW( zero, mf_h0, zero, mf_h1, zero, mf_h2, zero, mf_h3,
mf_vec0, mf_vec2, mf_vec4, mf_vec6 );
ILVL_H4_SW( zero, mf_h0, zero, mf_h1, zero, mf_h2, zero, mf_h3,
mf_vec1, mf_vec3, mf_vec5, mf_vec7 );
dct_w0 = __msa_add_a_w( dct_signed_w0, bias0 );
dct_w1 = __msa_add_a_w( dct_signed_w1, bias1 );
dct_w2 = __msa_add_a_w( dct_signed_w2, bias2 );
dct_w3 = __msa_add_a_w( dct_signed_w3, bias3 );
dct_w4 = __msa_add_a_w( dct_signed_w4, bias4 );
dct_w5 = __msa_add_a_w( dct_signed_w5, bias5 );
dct_w6 = __msa_add_a_w( dct_signed_w6, bias6 );
dct_w7 = __msa_add_a_w( dct_signed_w7, bias7 );
dct_w0 *= mf_vec0;
dct_w1 *= mf_vec1;
dct_w2 *= mf_vec2;
dct_w3 *= mf_vec3;
dct_w4 *= mf_vec4;
dct_w5 *= mf_vec5;
dct_w6 *= mf_vec6;
dct_w7 *= mf_vec7;
SRA_4V( dct_w0, dct_w1, dct_w2, dct_w3, 16 );
SRA_4V( dct_w4, dct_w5, dct_w6, dct_w7, 16 );
PCKEV_H4_SH( dct_w1, dct_w0, dct_w3, dct_w2, dct_w5, dct_w4, dct_w7, dct_w6,
dct_h0, dct_h1, dct_h2, dct_h3 );
SUB4( zero, dct_h0, zero, dct_h1, zero, dct_h2, zero, dct_h3,
dct0, dct1, dct2, dct3 );
dct0 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h0,
( v16u8 ) dct0, ( v16u8 ) dct0_mask );
dct1 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h1,
( v16u8 ) dct1, ( v16u8 ) dct1_mask );
dct2 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h2,
( v16u8 ) dct2, ( v16u8 ) dct2_mask );
dct3 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h3,
( v16u8 ) dct3, ( v16u8 ) dct3_mask );
non_zero = HADD_SW_S32( ( v4u32 )( dct_h0 + dct_h1 + dct_h2 + dct_h3 ) );
ST_SH4( dct0, dct1, dct2, dct3, p_dct, 8 );
LD_SH4( p_dct + 32, 8, dct0, dct1, dct2, dct3 );
dct0_mask = __msa_clei_s_h( dct0, 0 );
dct1_mask = __msa_clei_s_h( dct1, 0 );
dct2_mask = __msa_clei_s_h( dct2, 0 );
dct3_mask = __msa_clei_s_h( dct3, 0 );
UNPCK_SH_SW( dct0, dct_signed_w0, dct_signed_w1 );
UNPCK_SH_SW( dct1, dct_signed_w2, dct_signed_w3 );
UNPCK_SH_SW( dct2, dct_signed_w4, dct_signed_w5 );
UNPCK_SH_SW( dct3, dct_signed_w6, dct_signed_w7 );
LD_SH4( p_bias + 32, 8, bias_h0, bias_h1, bias_h2, bias_h3 );
ILVR_H4_SW( zero, bias_h0, zero, bias_h1, zero, bias_h2, zero, bias_h3,
bias0, bias2, bias4, bias6 );
ILVL_H4_SW( zero, bias_h0, zero, bias_h1, zero, bias_h2, zero, bias_h3,
bias1, bias3, bias5, bias7 );
LD_SH4( p_mf + 32, 8, mf_h0, mf_h1, mf_h2, mf_h3 );
ILVR_H4_SW( zero, mf_h0, zero, mf_h1, zero, mf_h2, zero, mf_h3,
mf_vec0, mf_vec2, mf_vec4, mf_vec6 );
ILVL_H4_SW( zero, mf_h0, zero, mf_h1, zero, mf_h2, zero, mf_h3,
mf_vec1, mf_vec3, mf_vec5, mf_vec7 );
dct_w0 = __msa_add_a_w( dct_signed_w0, bias0 );
dct_w1 = __msa_add_a_w( dct_signed_w1, bias1 );
dct_w2 = __msa_add_a_w( dct_signed_w2, bias2 );
dct_w3 = __msa_add_a_w( dct_signed_w3, bias3 );
dct_w4 = __msa_add_a_w( dct_signed_w4, bias4 );
dct_w5 = __msa_add_a_w( dct_signed_w5, bias5 );
dct_w6 = __msa_add_a_w( dct_signed_w6, bias6 );
dct_w7 = __msa_add_a_w( dct_signed_w7, bias7 );
dct_w0 *= mf_vec0;
dct_w1 *= mf_vec1;
dct_w2 *= mf_vec2;
dct_w3 *= mf_vec3;
dct_w4 *= mf_vec4;
dct_w5 *= mf_vec5;
dct_w6 *= mf_vec6;
dct_w7 *= mf_vec7;
SRA_4V( dct_w0, dct_w1, dct_w2, dct_w3, 16 );
SRA_4V( dct_w4, dct_w5, dct_w6, dct_w7, 16 );
PCKEV_H2_SH( dct_w1, dct_w0, dct_w3, dct_w2, dct_h0, dct_h1 );
PCKEV_H2_SH( dct_w5, dct_w4, dct_w7, dct_w6, dct_h2, dct_h3 );
SUB4( zero, dct_h0, zero, dct_h1, zero, dct_h2, zero, dct_h3,
dct0, dct1, dct2, dct3 );
dct0 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h0,
( v16u8 ) dct0, ( v16u8 ) dct0_mask );
dct1 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h1,
( v16u8 ) dct1, ( v16u8 ) dct1_mask );
dct2 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h2,
( v16u8 ) dct2, ( v16u8 ) dct2_mask );
dct3 = ( v8i16 ) __msa_bmnz_v( ( v16u8 ) dct_h3,
( v16u8 ) dct3, ( v16u8 ) dct3_mask );
non_zero += HADD_SW_S32( ( v4u32 ) ( dct_h0 + dct_h1 + dct_h2 + dct_h3 ) );
ST_SH4( dct0, dct1, dct2, dct3, p_dct + 32, 8 );
return !!non_zero;
}
