static uint8x8_t
paeth(uint8x8_t a, uint8x8_t b, uint8x8_t c)
{
uint8x8_t d, e;
uint16x8_t p1, pa, pb, pc;
p1 = vaddl_u8(a, b); /* a + b */
pc = vaddl_u8(c, c); /* c * 2 */
pa = vabdl_u8(b, c); /* pa */
pb = vabdl_u8(a, c); /* pb */
pc = vabdq_u16(p1, pc); /* pc */
p1 = vcleq_u16(pa, pb); /* pa <= pb */
pa = vcleq_u16(pa, pc); /* pa <= pc */
pb = vcleq_u16(pb, pc); /* pb <= pc */
p1 = vandq_u16(p1, pa); /* pa <= pb && pa <= pc */
d = vmovn_u16(pb);
e = vmovn_u16(p1);
d = vbsl_u8(d, b, c);
e = vbsl_u8(e, a, d);
return e;
}
unsigned int vp9_avg_8x8_neon(const uint8_t *s, int p) {
uint8x8_t v_s0 = vld1_u8(s);
const uint8x8_t v_s1 = vld1_u8(s + p);
uint16x8_t v_sum = vaddl_u8(v_s0, v_s1);
v_s0 = vld1_u8(s + 2 * p);
v_sum = vaddw_u8(v_sum, v_s0);
v_s0 = vld1_u8(s + 3 * p);
v_sum = vaddw_u8(v_sum, v_s0);
v_s0 = vld1_u8(s + 4 * p);
v_sum = vaddw_u8(v_sum, v_s0);
v_s0 = vld1_u8(s + 5 * p);
v_sum = vaddw_u8(v_sum, v_s0);
v_s0 = vld1_u8(s + 6 * p);
v_sum = vaddw_u8(v_sum, v_s0);
v_s0 = vld1_u8(s + 7 * p);
v_sum = vaddw_u8(v_sum, v_s0);
return (horizontal_add_u16x8(v_sum) + 32) >> 6;
}
static WEBP_INLINE uint32_t ClampedAddSubtractFull(const uint32_t* const c0,
const uint32_t* const c1,
const uint32_t* const c2) {
const uint8x8_t p0 = vreinterpret_u8_u64(vcreate_u64(*c0));
const uint8x8_t p1 = vreinterpret_u8_u64(vcreate_u64(*c1));
const uint8x8_t p2 = vreinterpret_u8_u64(vcreate_u64(*c2));
const uint16x8_t sum0 = vaddl_u8(p0, p1); // add and widen
const uint16x8_t sum1 = vqsubq_u16(sum0, vmovl_u8(p2)); // widen and subtract
const uint8x8_t out = vqmovn_u16(sum1); // narrow and clamp
return vget_lane_u32(vreinterpret_u32_u8(out), 0);
}
unsigned int aom_avg_4x4_neon(const uint8_t *s, int p) {
uint16x8_t v_sum;
uint32x2_t v_s0 = vdup_n_u32(0);
uint32x2_t v_s1 = vdup_n_u32(0);
v_s0 = vld1_lane_u32((const uint32_t *)s, v_s0, 0);
v_s0 = vld1_lane_u32((const uint32_t *)(s + p), v_s0, 1);
v_s1 = vld1_lane_u32((const uint32_t *)(s + 2 * p), v_s1, 0);
v_s1 = vld1_lane_u32((const uint32_t *)(s + 3 * p), v_s1, 1);
v_sum = vaddl_u8(vreinterpret_u8_u32(v_s0), vreinterpret_u8_u32(v_s1));
return (horizontal_add_u16x8(v_sum) + 8) >> 4;
}
static WEBP_INLINE uint32_t ClampedAddSubtractFull(const uint32_t* const c0,
const uint32_t* const c1,
const uint32_t* const c2) {
const uint64x1_t C0 = { *c0, 0 }, C1 = { *c1, 0 }, C2 = { *c2, 0 };
const uint8x8_t p0 = vreinterpret_u8_u64(C0);
const uint8x8_t p1 = vreinterpret_u8_u64(C1);
const uint8x8_t p2 = vreinterpret_u8_u64(C2);
const uint16x8_t sum0 = vaddl_u8(p0, p1); // add and widen
const uint16x8_t sum1 = vqsubq_u16(sum0, vmovl_u8(p2)); // widen and subtract
const uint8x8_t out = vqmovn_u16(sum1); // narrow and clamp
uint32_t ret;
vst1_lane_u32(&ret, vreinterpret_u32_u8(out), 0);
return ret;
}
template <> SIMD_INLINE uint16x8_t ReduceColTail<false>(const uint8_t *src)
{
const uint8x8x2_t t01 = vld2_u8(src - 1);
const uint8x8x2_t t23 = Deinterleave(LoadAfterLast<1>(LoadAfterLast<1>(vld1q_u8(src - 1))));
return vaddq_u16(vaddl_u8(t01.val[0], t23.val[1]), vmulq_u16(vaddl_u8(t01.val[1], t23.val[0]), K16_0003));
}
SIMD_INLINE uint16x8_t ReduceColBody(const uint8_t *src)
{
const uint8x8x2_t t01 = vld2_u8(src - 1);
const uint8x8x2_t t23 = vld2_u8(src + 1);
return vaddq_u16(vaddl_u8(t01.val[0], t23.val[1]), vmulq_u16(vaddl_u8(t01.val[1], t23.val[0]), K16_0003));
}
SIMD_INLINE uint16x8_t ReduceColNose(const uint8_t * src)
{
const uint8x8x2_t t01 = Deinterleave(LoadBeforeFirst<1>(vld1q_u8(src)));
const uint8x8x2_t t23 = vld2_u8(src + 1);
return vaddq_u16(vaddl_u8(t01.val[0], t23.val[1]), vmulq_u16(vaddl_u8(t01.val[1], t23.val[0]), K16_0003));
}
static INLINE void mbloop_filter_neon(uint8x8_t dblimit, // mblimit
uint8x8_t dlimit, // limit
uint8x8_t dthresh, // thresh
uint8x8_t d3u8, // p2
uint8x8_t d4u8, // p2
uint8x8_t d5u8, // p1
uint8x8_t d6u8, // p0
uint8x8_t d7u8, // q0
uint8x8_t d16u8, // q1
uint8x8_t d17u8, // q2
uint8x8_t d18u8, // q3
uint8x8_t *d0ru8, // p1
uint8x8_t *d1ru8, // p1
uint8x8_t *d2ru8, // p0
uint8x8_t *d3ru8, // q0
uint8x8_t *d4ru8, // q1
uint8x8_t *d5ru8) { // q1
uint32_t flat;
uint8x8_t d0u8, d1u8, d2u8, d19u8, d20u8, d21u8, d22u8, d23u8, d24u8;
uint8x8_t d25u8, d26u8, d27u8, d28u8, d29u8, d30u8, d31u8;
int16x8_t q15s16;
uint16x8_t q10u16, q14u16;
int8x8_t d21s8, d24s8, d25s8, d26s8, d28s8, d29s8, d30s8;
d19u8 = vabd_u8(d3u8, d4u8);
d20u8 = vabd_u8(d4u8, d5u8);
d21u8 = vabd_u8(d5u8, d6u8);
d22u8 = vabd_u8(d16u8, d7u8);
d23u8 = vabd_u8(d17u8, d16u8);
d24u8 = vabd_u8(d18u8, d17u8);
d19u8 = vmax_u8(d19u8, d20u8);
d20u8 = vmax_u8(d21u8, d22u8);
d25u8 = vabd_u8(d6u8, d4u8);
d23u8 = vmax_u8(d23u8, d24u8);
d26u8 = vabd_u8(d7u8, d17u8);
d19u8 = vmax_u8(d19u8, d20u8);
d24u8 = vabd_u8(d6u8, d7u8);
d27u8 = vabd_u8(d3u8, d6u8);
d28u8 = vabd_u8(d18u8, d7u8);
d19u8 = vmax_u8(d19u8, d23u8);
d23u8 = vabd_u8(d5u8, d16u8);
d24u8 = vqadd_u8(d24u8, d24u8);
d19u8 = vcge_u8(dlimit, d19u8);
d25u8 = vmax_u8(d25u8, d26u8);
d26u8 = vmax_u8(d27u8, d28u8);
d23u8 = vshr_n_u8(d23u8, 1);
d25u8 = vmax_u8(d25u8, d26u8);
d24u8 = vqadd_u8(d24u8, d23u8);
d20u8 = vmax_u8(d20u8, d25u8);
d23u8 = vdup_n_u8(1);
d24u8 = vcge_u8(dblimit, d24u8);
d21u8 = vcgt_u8(d21u8, dthresh);
d20u8 = vcge_u8(d23u8, d20u8);
d19u8 = vand_u8(d19u8, d24u8);
d23u8 = vcgt_u8(d22u8, dthresh);
d20u8 = vand_u8(d20u8, d19u8);
d22u8 = vdup_n_u8(0x80);
d23u8 = vorr_u8(d21u8, d23u8);
q10u16 = vcombine_u16(vreinterpret_u16_u8(d20u8), vreinterpret_u16_u8(d21u8));
d30u8 = vshrn_n_u16(q10u16, 4);
flat = vget_lane_u32(vreinterpret_u32_u8(d30u8), 0);
if (flat == 0xffffffff) { // Check for all 1's, power_branch_only
d27u8 = vdup_n_u8(3);
d21u8 = vdup_n_u8(2);
q14u16 = vaddl_u8(d6u8, d7u8);
q14u16 = vmlal_u8(q14u16, d3u8, d27u8);
q14u16 = vmlal_u8(q14u16, d4u8, d21u8);
q14u16 = vaddw_u8(q14u16, d5u8);
*d0ru8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d3u8);
q14u16 = vsubw_u8(q14u16, d4u8);
q14u16 = vaddw_u8(q14u16, d5u8);
q14u16 = vaddw_u8(q14u16, d16u8);
*d1ru8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d3u8);
q14u16 = vsubw_u8(q14u16, d5u8);
q14u16 = vaddw_u8(q14u16, d6u8);
q14u16 = vaddw_u8(q14u16, d17u8);
*d2ru8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d3u8);
q14u16 = vsubw_u8(q14u16, d6u8);
q14u16 = vaddw_u8(q14u16, d7u8);
q14u16 = vaddw_u8(q14u16, d18u8);
*d3ru8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d4u8);
q14u16 = vsubw_u8(q14u16, d7u8);
q14u16 = vaddw_u8(q14u16, d16u8);
q14u16 = vaddw_u8(q14u16, d18u8);
*d4ru8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d5u8);
q14u16 = vsubw_u8(q14u16, d16u8);
q14u16 = vaddw_u8(q14u16, d17u8);
q14u16 = vaddw_u8(q14u16, d18u8);
*d5ru8 = vqrshrn_n_u16(q14u16, 3);
} else {
d21u8 = veor_u8(d7u8, d22u8);
d24u8 = veor_u8(d6u8, d22u8);
d25u8 = veor_u8(d5u8, d22u8);
d26u8 = veor_u8(d16u8, d22u8);
d27u8 = vdup_n_u8(3);
d28s8 = vsub_s8(vreinterpret_s8_u8(d21u8), vreinterpret_s8_u8(d24u8));
d29s8 = vqsub_s8(vreinterpret_s8_u8(d25u8), vreinterpret_s8_u8(d26u8));
q15s16 = vmull_s8(d28s8, vreinterpret_s8_u8(d27u8));
d29s8 = vand_s8(d29s8, vreinterpret_s8_u8(d23u8));
q15s16 = vaddw_s8(q15s16, d29s8);
d29u8 = vdup_n_u8(4);
d28s8 = vqmovn_s16(q15s16);
d28s8 = vand_s8(d28s8, vreinterpret_s8_u8(d19u8));
d30s8 = vqadd_s8(d28s8, vreinterpret_s8_u8(d27u8));
d29s8 = vqadd_s8(d28s8, vreinterpret_s8_u8(d29u8));
d30s8 = vshr_n_s8(d30s8, 3);
d29s8 = vshr_n_s8(d29s8, 3);
d24s8 = vqadd_s8(vreinterpret_s8_u8(d24u8), d30s8);
d21s8 = vqsub_s8(vreinterpret_s8_u8(d21u8), d29s8);
d29s8 = vrshr_n_s8(d29s8, 1);
d29s8 = vbic_s8(d29s8, vreinterpret_s8_u8(d23u8));
d25s8 = vqadd_s8(vreinterpret_s8_u8(d25u8), d29s8);
d26s8 = vqsub_s8(vreinterpret_s8_u8(d26u8), d29s8);
if (flat == 0) { // filter_branch_only
*d0ru8 = d4u8;
*d1ru8 = veor_u8(vreinterpret_u8_s8(d25s8), d22u8);
*d2ru8 = veor_u8(vreinterpret_u8_s8(d24s8), d22u8);
*d3ru8 = veor_u8(vreinterpret_u8_s8(d21s8), d22u8);
*d4ru8 = veor_u8(vreinterpret_u8_s8(d26s8), d22u8);
*d5ru8 = d17u8;
return;
}
d21u8 = veor_u8(vreinterpret_u8_s8(d21s8), d22u8);
d24u8 = veor_u8(vreinterpret_u8_s8(d24s8), d22u8);
d25u8 = veor_u8(vreinterpret_u8_s8(d25s8), d22u8);
d26u8 = veor_u8(vreinterpret_u8_s8(d26s8), d22u8);
d23u8 = vdup_n_u8(2);
q14u16 = vaddl_u8(d6u8, d7u8);
q14u16 = vmlal_u8(q14u16, d3u8, d27u8);
q14u16 = vmlal_u8(q14u16, d4u8, d23u8);
d0u8 = vbsl_u8(d20u8, dblimit, d4u8);
q14u16 = vaddw_u8(q14u16, d5u8);
d1u8 = vbsl_u8(d20u8, dlimit, d25u8);
d30u8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d3u8);
q14u16 = vsubw_u8(q14u16, d4u8);
q14u16 = vaddw_u8(q14u16, d5u8);
q14u16 = vaddw_u8(q14u16, d16u8);
d2u8 = vbsl_u8(d20u8, dthresh, d24u8);
d31u8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d3u8);
q14u16 = vsubw_u8(q14u16, d5u8);
q14u16 = vaddw_u8(q14u16, d6u8);
q14u16 = vaddw_u8(q14u16, d17u8);
*d0ru8 = vbsl_u8(d20u8, d30u8, d0u8);
d23u8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d3u8);
q14u16 = vsubw_u8(q14u16, d6u8);
q14u16 = vaddw_u8(q14u16, d7u8);
*d1ru8 = vbsl_u8(d20u8, d31u8, d1u8);
q14u16 = vaddw_u8(q14u16, d18u8);
*d2ru8 = vbsl_u8(d20u8, d23u8, d2u8);
d22u8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d4u8);
q14u16 = vsubw_u8(q14u16, d7u8);
q14u16 = vaddw_u8(q14u16, d16u8);
d3u8 = vbsl_u8(d20u8, d3u8, d21u8);
q14u16 = vaddw_u8(q14u16, d18u8);
d4u8 = vbsl_u8(d20u8, d4u8, d26u8);
d6u8 = vqrshrn_n_u16(q14u16, 3);
q14u16 = vsubw_u8(q14u16, d5u8);
q14u16 = vsubw_u8(q14u16, d16u8);
q14u16 = vaddw_u8(q14u16, d17u8);
q14u16 = vaddw_u8(q14u16, d18u8);
d5u8 = vbsl_u8(d20u8, d5u8, d17u8);
d7u8 = vqrshrn_n_u16(q14u16, 3);
*d3ru8 = vbsl_u8(d20u8, d22u8, d3u8);
*d4ru8 = vbsl_u8(d20u8, d6u8, d4u8);
*d5ru8 = vbsl_u8(d20u8, d7u8, d5u8);
}
return;
}
void UpsampleRgbaLinePairNEON(const uint8_t *top_y, const uint8_t *bottom_y, const uint8_t *top_u, const uint8_t *top_v, const uint8_t *cur_u, const uint8_t *cur_v, uint8_t *top_dst, uint8_t *bottom_dst, int len)
{
int block;
uint8_t uv_buf[2 * 32 + 15];
uint8_t *const r_uv = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15);
const int uv_len = (len + 1) >> 1;
const int num_blocks = (uv_len - 1) >> 3;
const int leftover = uv_len - num_blocks * 8;
const int last_pos = 1 + 16 * num_blocks;
const int u_diag = ((top_u[0] + cur_u[0]) >> 1) + 1;
const int v_diag = ((top_v[0] + cur_v[0]) >> 1) + 1;
const int16x4_t cf16 = vld1_s16(coef);
const int32x2_t cf32 = vmov_n_s32(76283);
const uint8x8_t u16 = vmov_n_u8(16);
const uint8x8_t u128 = vmov_n_u8(128);
for (block = 0; block < num_blocks; ++block) {
{
uint8x8_t a = vld1_u8(top_u);
uint8x8_t b = vld1_u8(top_u + 1);
uint8x8_t c = vld1_u8(cur_u);
uint8x8_t d = vld1_u8(cur_u + 1);
uint16x8_t al = vshll_n_u8(a, 1);
uint16x8_t bl = vshll_n_u8(b, 1);
uint16x8_t cl = vshll_n_u8(c, 1);
uint16x8_t dl = vshll_n_u8(d, 1);
uint8x8_t diag1, diag2;
uint16x8_t sl;
sl = vaddl_u8(a, b);
sl = vaddw_u8(sl, c);
sl = vaddw_u8(sl, d);
al = vaddq_u16(sl, al);
bl = vaddq_u16(sl, bl);
al = vaddq_u16(al, dl);
bl = vaddq_u16(bl, cl);
diag2 = vshrn_n_u16(al, 3);
diag1 = vshrn_n_u16(bl, 3);
a = vrhadd_u8(a, diag1);
b = vrhadd_u8(b, diag2);
c = vrhadd_u8(c, diag2);
d = vrhadd_u8(d, diag1);
{
const uint8x8x2_t a_b = {{ a, b }};
const uint8x8x2_t c_d = {{ c, d }};
vst2_u8(r_uv, a_b);
vst2_u8(r_uv + 32, c_d);
}
}
{
uint8x8_t a = vld1_u8(top_v);
uint8x8_t b = vld1_u8(top_v + 1);
uint8x8_t c = vld1_u8(cur_v);
uint8x8_t d = vld1_u8(cur_v + 1);
uint16x8_t al = vshll_n_u8(a, 1);
uint16x8_t bl = vshll_n_u8(b, 1);
uint16x8_t cl = vshll_n_u8(c, 1);
uint16x8_t dl = vshll_n_u8(d, 1);
uint8x8_t diag1, diag2;
uint16x8_t sl;
sl = vaddl_u8(a, b);
sl = vaddw_u8(sl, c);
sl = vaddw_u8(sl, d);
al = vaddq_u16(sl, al);
bl = vaddq_u16(sl, bl);
al = vaddq_u16(al, dl);
bl = vaddq_u16(bl, cl);
diag2 = vshrn_n_u16(al, 3);
diag1 = vshrn_n_u16(bl, 3);
a = vrhadd_u8(a, diag1);
b = vrhadd_u8(b, diag2);
c = vrhadd_u8(c, diag2);
d = vrhadd_u8(d, diag1);
{
const uint8x8x2_t a_b = {{ a, b }};
const uint8x8x2_t c_d = {{ c, d }};
vst2_u8(r_uv + 16, a_b);
vst2_u8(r_uv + 16 + 32, c_d);
}
}
{
if (top_y) {
{
int i;
for (i = 0; i < 16; i += 8) {
int off = ((16 * block + 1) + i) * 4;
uint8x8_t y = vld1_u8(top_y + (16 * block + 1) + i);
uint8x8_t u = vld1_u8((r_uv) + i);
uint8x8_t v = vld1_u8((r_uv) + i + 16);
int16x8_t yy = vreinterpretq_s16_u16(vsubl_u8(y, u16));
int16x8_t uu = vreinterpretq_s16_u16(vsubl_u8(u, u128));
int16x8_t vv = vreinterpretq_s16_u16(vsubl_u8(v, u128));
int16x8_t ud = vshlq_n_s16(uu, 1);
int16x8_t vd = vshlq_n_s16(vv, 1);
int32x4_t vrl = vqdmlal_lane_s16(vshll_n_s16(vget_low_s16(vv), 1), vget_low_s16(vd), cf16, 0);
int32x4_t vrh = vqdmlal_lane_s16(vshll_n_s16(vget_high_s16(vv), 1), vget_high_s16(vd), cf16, 0);
int16x8_t vr = vcombine_s16(vrshrn_n_s32(vrl, 16), vrshrn_n_s32(vrh, 16));
int32x4_t vl = vmovl_s16(vget_low_s16(vv));
int32x4_t vh = vmovl_s16(vget_high_s16(vv));
int32x4_t ugl = vmlal_lane_s16(vl, vget_low_s16(uu), cf16, 1);
int32x4_t ugh = vmlal_lane_s16(vh, vget_high_s16(uu), cf16, 1);
int32x4_t gcl = vqdmlal_lane_s16(ugl, vget_low_s16(vv), cf16, 2);
int32x4_t gch = vqdmlal_lane_s16(ugh, vget_high_s16(vv), cf16, 2);
int16x8_t gc = vcombine_s16(vrshrn_n_s32(gcl, 16), vrshrn_n_s32(gch, 16));
int32x4_t ubl = vqdmlal_lane_s16(vshll_n_s16(vget_low_s16(uu), 1), vget_low_s16(ud), cf16, 3);
int32x4_t ubh = vqdmlal_lane_s16(vshll_n_s16(vget_high_s16(uu), 1), vget_high_s16(ud), cf16, 3);
int16x8_t ub = vcombine_s16(vrshrn_n_s32(ubl, 16), vrshrn_n_s32(ubh, 16));
int32x4_t rl = vaddl_s16(vget_low_s16(yy), vget_low_s16(vr));
int32x4_t rh = vaddl_s16(vget_high_s16(yy), vget_high_s16(vr));
int32x4_t gl = vsubl_s16(vget_low_s16(yy), vget_low_s16(gc));
int32x4_t gh = vsubl_s16(vget_high_s16(yy), vget_high_s16(gc));
int32x4_t bl = vaddl_s16(vget_low_s16(yy), vget_low_s16(ub));
int32x4_t bh = vaddl_s16(vget_high_s16(yy), vget_high_s16(ub));
rl = vmulq_lane_s32(rl, cf32, 0);
rh = vmulq_lane_s32(rh, cf32, 0);
gl = vmulq_lane_s32(gl, cf32, 0);
gh = vmulq_lane_s32(gh, cf32, 0);
bl = vmulq_lane_s32(bl, cf32, 0);
bh = vmulq_lane_s32(bh, cf32, 0);
y = vqmovun_s16(vcombine_s16(vrshrn_n_s32(rl, 16), vrshrn_n_s32(rh, 16)));
u = vqmovun_s16(vcombine_s16(vrshrn_n_s32(gl, 16), vrshrn_n_s32(gh, 16)));
v = vqmovun_s16(vcombine_s16(vrshrn_n_s32(bl, 16), vrshrn_n_s32(bh, 16)));
do {
const uint8x8x4_t r_g_b_v255 = {{ y, u, v, vmov_n_u8(255) }};
vst4_u8(top_dst + off, r_g_b_v255);
} while (0);
}
}
}
if (bottom_y) {
{
int i;
for (i = 0; i < 16; i += 8) {
int off = ((16 * block + 1) + i) * 4;
uint8x8_t y = vld1_u8(bottom_y + (16 * block + 1) + i);
uint8x8_t u = vld1_u8(((r_uv) + 32) + i);
uint8x8_t v = vld1_u8(((r_uv) + 32) + i + 16);
int16x8_t yy = vreinterpretq_s16_u16(vsubl_u8(y, u16));
int16x8_t uu = vreinterpretq_s16_u16(vsubl_u8(u, u128));
int16x8_t vv = vreinterpretq_s16_u16(vsubl_u8(v, u128));
int16x8_t ud = vshlq_n_s16(uu, 1);
int16x8_t vd = vshlq_n_s16(vv, 1);
int32x4_t vrl = vqdmlal_lane_s16(vshll_n_s16(vget_low_s16(vv), 1), vget_low_s16(vd), cf16, 0);
int32x4_t vrh = vqdmlal_lane_s16(vshll_n_s16(vget_high_s16(vv), 1), vget_high_s16(vd), cf16, 0);
int16x8_t vr = vcombine_s16(vrshrn_n_s32(vrl, 16), vrshrn_n_s32(vrh, 16));
int32x4_t vl = vmovl_s16(vget_low_s16(vv));
int32x4_t vh = vmovl_s16(vget_high_s16(vv));
int32x4_t ugl = vmlal_lane_s16(vl, vget_low_s16(uu), cf16, 1);
int32x4_t ugh = vmlal_lane_s16(vh, vget_high_s16(uu), cf16, 1);
int32x4_t gcl = vqdmlal_lane_s16(ugl, vget_low_s16(vv), cf16, 2);
int32x4_t gch = vqdmlal_lane_s16(ugh, vget_high_s16(vv), cf16, 2);
int16x8_t gc = vcombine_s16(vrshrn_n_s32(gcl, 16), vrshrn_n_s32(gch, 16));
int32x4_t ubl = vqdmlal_lane_s16(vshll_n_s16(vget_low_s16(uu), 1), vget_low_s16(ud), cf16, 3);
int32x4_t ubh = vqdmlal_lane_s16(vshll_n_s16(vget_high_s16(uu), 1), vget_high_s16(ud), cf16, 3);
int16x8_t ub = vcombine_s16(vrshrn_n_s32(ubl, 16), vrshrn_n_s32(ubh, 16));
int32x4_t rl = vaddl_s16(vget_low_s16(yy), vget_low_s16(vr));
int32x4_t rh = vaddl_s16(vget_high_s16(yy), vget_high_s16(vr));
int32x4_t gl = vsubl_s16(vget_low_s16(yy), vget_low_s16(gc));
int32x4_t gh = vsubl_s16(vget_high_s16(yy), vget_high_s16(gc));
int32x4_t bl = vaddl_s16(vget_low_s16(yy), vget_low_s16(ub));
int32x4_t bh = vaddl_s16(vget_high_s16(yy), vget_high_s16(ub));
rl = vmulq_lane_s32(rl, cf32, 0);
rh = vmulq_lane_s32(rh, cf32, 0);
gl = vmulq_lane_s32(gl, cf32, 0);
gh = vmulq_lane_s32(gh, cf32, 0);
bl = vmulq_lane_s32(bl, cf32, 0);
bh = vmulq_lane_s32(bh, cf32, 0);
y = vqmovun_s16(vcombine_s16(vrshrn_n_s32(rl, 16), vrshrn_n_s32(rh, 16)));
u = vqmovun_s16(vcombine_s16(vrshrn_n_s32(gl, 16), vrshrn_n_s32(gh, 16)));
v = vqmovun_s16(vcombine_s16(vrshrn_n_s32(bl, 16), vrshrn_n_s32(bh, 16)));
do {
const uint8x8x4_t r_g_b_v255 = {{ y, u, v, vmov_n_u8(255) }};
vst4_u8(bottom_dst + off, r_g_b_v255);
} while (0);
}
}
}
}
}
}