static inline void char_to_float_vectors(const unsigned char * sourcep,
float32x4_t *mp0, float32x4_t * mp1)
{
uint8x8_t rawpixels; /* source pixels as {[YUYV]0 [YUYV]1} */
int16x8_t widerpixels; /* rawpixels promoted to shorts per component */
int16x4_t high16, low16;
int32x4_t high32, low32;
const int16x8_t uvbias = {0, 128, 0, 128, 0, 128, 0, 128};
rawpixels = vld1_u8(sourcep);
widerpixels = vreinterpretq_s16_u16(vmovl_u8(rawpixels));
/* subtract uvbias from widerpixels */
widerpixels = vsubq_s16(widerpixels, uvbias);
/* now take widerpixels apart into (low16, high16) and */
/* then expand those into (low32, high32) */
low16 = vget_low_s16(widerpixels);
high16 = vget_high_s16(widerpixels);
high32 = vmovl_s16(high16);
low32 = vmovl_s16(low16);
/* now convert low32 and high32 into floats and store them in */
/* *mp0, *mp1 */
*mp0 = vcvtq_f32_s32(low32);
*mp1 = vcvtq_f32_s32(high32);
}
int normL1_(const uchar* a, const uchar* b, int n)
{
int j = 0, d = 0;
#if CV_SSE
__m128i d0 = _mm_setzero_si128();
for( ; j <= n - 16; j += 16 )
{
__m128i t0 = _mm_loadu_si128((const __m128i*)(a + j));
__m128i t1 = _mm_loadu_si128((const __m128i*)(b + j));
d0 = _mm_add_epi32(d0, _mm_sad_epu8(t0, t1));
}
for( ; j <= n - 4; j += 4 )
{
__m128i t0 = _mm_cvtsi32_si128(*(const int*)(a + j));
__m128i t1 = _mm_cvtsi32_si128(*(const int*)(b + j));
d0 = _mm_add_epi32(d0, _mm_sad_epu8(t0, t1));
}
d = _mm_cvtsi128_si32(_mm_add_epi32(d0, _mm_unpackhi_epi64(d0, d0)));
#elif CV_NEON
uint32x4_t v_sum = vdupq_n_u32(0.0f);
for ( ; j <= n - 16; j += 16)
{
uint8x16_t v_dst = vabdq_u8(vld1q_u8(a + j), vld1q_u8(b + j));
uint16x8_t v_low = vmovl_u8(vget_low_u8(v_dst)), v_high = vmovl_u8(vget_high_u8(v_dst));
v_sum = vaddq_u32(v_sum, vaddl_u16(vget_low_u16(v_low), vget_low_u16(v_high)));
v_sum = vaddq_u32(v_sum, vaddl_u16(vget_high_u16(v_low), vget_high_u16(v_high)));
}
uint CV_DECL_ALIGNED(16) buf[4];
vst1q_u32(buf, v_sum);
d = buf[0] + buf[1] + buf[2] + buf[3];
#endif
{
for( ; j <= n - 4; j += 4 )
{
d += std::abs(a[j] - b[j]) + std::abs(a[j+1] - b[j+1]) +
std::abs(a[j+2] - b[j+2]) + std::abs(a[j+3] - b[j+3]);
}
}
for( ; j < n; j++ )
d += std::abs(a[j] - b[j]);
return d;
}
void byte2word64_neon(const uint8_t *t, const int pitch, float *pf) {
uint16_t *p = (uint16_t *)pf;
vst1q_u16(p, vmovl_u8(vld1_u8(t)));
vst1q_u16(p + 8, vmovl_u8(vld1_u8(t + 8)));
vst1q_u16(p + 16, vmovl_u8(vld1_u8(t + pitch * 2)));
vst1q_u16(p + 24, vmovl_u8(vld1_u8(t + pitch * 2 + 8)));
vst1q_u16(p + 32, vmovl_u8(vld1_u8(t + pitch * 4)));
vst1q_u16(p + 40, vmovl_u8(vld1_u8(t + pitch * 4 + 8)));
vst1q_u16(p + 48, vmovl_u8(vld1_u8(t + pitch * 6)));
vst1q_u16(p + 56, vmovl_u8(vld1_u8(t + pitch * 6 + 8)));
}
static WEBP_INLINE uint32_t ClampedAddSubtractFull(const uint32_t* const c0,
const uint32_t* const c1,
const uint32_t* const c2) {
const uint8x8_t p0 = vreinterpret_u8_u64(vcreate_u64(*c0));
const uint8x8_t p1 = vreinterpret_u8_u64(vcreate_u64(*c1));
const uint8x8_t p2 = vreinterpret_u8_u64(vcreate_u64(*c2));
const uint16x8_t sum0 = vaddl_u8(p0, p1); // add and widen
const uint16x8_t sum1 = vqsubq_u16(sum0, vmovl_u8(p2)); // widen and subtract
const uint8x8_t out = vqmovn_u16(sum1); // narrow and clamp
return vget_lane_u32(vreinterpret_u32_u8(out), 0);
}
void byte2float48_neon(const uint8_t *t, const int pitch, float *p) {
uint16x8_t m0, m1, m2, m3, m4, m5;
uint32x2_t temp1, temp4;
m0 = vmovl_u8(vld1_u8(t));
temp1 = vld1_lane_u32((const uint32_t *)(t + 8), temp1, 0);
temp1 = vld1_lane_u32((const uint32_t *)(t + pitch * 2), temp1, 1);
m1 = vmovl_u8(vreinterpret_u8_u32(temp1));
m2 = vmovl_u8(vld1_u8(t + pitch * 2 + 4));
t += pitch * 4;
m3 = vmovl_u8(vld1_u8(t));
temp4 = vld1_lane_u32((const uint32_t *)(t + 8), temp4, 0);
temp4 = vld1_lane_u32((const uint32_t *)(t + pitch * 2), temp4, 1);
m4 = vmovl_u8(vreinterpret_u8_u32(temp4));
m5 = vmovl_u8(vld1_u8(t + pitch * 2 + 4));
vst1q_f32(p, vcvtq_f32_u32(vmovl_u16(vget_low_u16(m0))));
vst1q_f32(p + 4, vcvtq_f32_u32(vmovl_u16(vget_high_u16(m0))));
vst1q_f32(p + 8, vcvtq_f32_u32(vmovl_u16(vget_low_u16(m1))));
vst1q_f32(p + 12, vcvtq_f32_u32(vmovl_u16(vget_high_u16(m1))));
vst1q_f32(p + 16, vcvtq_f32_u32(vmovl_u16(vget_low_u16(m2))));
vst1q_f32(p + 20, vcvtq_f32_u32(vmovl_u16(vget_high_u16(m2))));
vst1q_f32(p + 24, vcvtq_f32_u32(vmovl_u16(vget_low_u16(m3))));
vst1q_f32(p + 28, vcvtq_f32_u32(vmovl_u16(vget_high_u16(m3))));
vst1q_f32(p + 32, vcvtq_f32_u32(vmovl_u16(vget_low_u16(m4))));
vst1q_f32(p + 36, vcvtq_f32_u32(vmovl_u16(vget_high_u16(m4))));
vst1q_f32(p + 40, vcvtq_f32_u32(vmovl_u16(vget_low_u16(m5))));
vst1q_f32(p + 44, vcvtq_f32_u32(vmovl_u16(vget_high_u16(m5))));
}
static WEBP_INLINE uint32_t ClampedAddSubtractFull(const uint32_t* const c0,
const uint32_t* const c1,
const uint32_t* const c2) {
const uint64x1_t C0 = { *c0, 0 }, C1 = { *c1, 0 }, C2 = { *c2, 0 };
const uint8x8_t p0 = vreinterpret_u8_u64(C0);
const uint8x8_t p1 = vreinterpret_u8_u64(C1);
const uint8x8_t p2 = vreinterpret_u8_u64(C2);
const uint16x8_t sum0 = vaddl_u8(p0, p1); // add and widen
const uint16x8_t sum1 = vqsubq_u16(sum0, vmovl_u8(p2)); // widen and subtract
const uint8x8_t out = vqmovn_u16(sum1); // narrow and clamp
uint32_t ret;
vst1_lane_u32(&ret, vreinterpret_u32_u8(out), 0);
return ret;
}
static uint8x8_t ConvertRGBToY_NEON(const uint8x8_t R,
const uint8x8_t G,
const uint8x8_t B) {
const uint16x8_t r = vmovl_u8(R);
const uint16x8_t g = vmovl_u8(G);
const uint16x8_t b = vmovl_u8(B);
const uint16x4_t r_lo = vget_low_u16(r);
const uint16x4_t r_hi = vget_high_u16(r);
const uint16x4_t g_lo = vget_low_u16(g);
const uint16x4_t g_hi = vget_high_u16(g);
const uint16x4_t b_lo = vget_low_u16(b);
const uint16x4_t b_hi = vget_high_u16(b);
const uint32x4_t tmp0_lo = vmull_n_u16( r_lo, 16839u);
const uint32x4_t tmp0_hi = vmull_n_u16( r_hi, 16839u);
const uint32x4_t tmp1_lo = vmlal_n_u16(tmp0_lo, g_lo, 33059u);
const uint32x4_t tmp1_hi = vmlal_n_u16(tmp0_hi, g_hi, 33059u);
const uint32x4_t tmp2_lo = vmlal_n_u16(tmp1_lo, b_lo, 6420u);
const uint32x4_t tmp2_hi = vmlal_n_u16(tmp1_hi, b_hi, 6420u);
const uint16x8_t Y1 = vcombine_u16(vrshrn_n_u32(tmp2_lo, 16),
vrshrn_n_u32(tmp2_hi, 16));
const uint16x8_t Y2 = vaddq_u16(Y1, vdupq_n_u16(16));
return vqmovn_u16(Y2);
}
static INLINE void scaledconvolve_vert_w4(
const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
const int y0_q4, const int y_step_q4, const int w, const int h) {
int y;
int y_q4 = y0_q4;
src -= src_stride * (SUBPEL_TAPS / 2 - 1);
y = h;
do {
const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
if (y_q4 & SUBPEL_MASK) {
const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
const int16x4_t filter3 = vdup_lane_s16(vget_low_s16(filters), 3);
const int16x4_t filter4 = vdup_lane_s16(vget_high_s16(filters), 0);
uint8x8_t s[8], d;
int16x4_t t[8], tt;
load_u8_8x8(src_y, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
&s[6], &s[7]);
t[0] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[0])));
t[1] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[1])));
t[2] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[2])));
t[3] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[3])));
t[4] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[4])));
t[5] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[5])));
t[6] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[6])));
t[7] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[7])));
tt = convolve8_4(t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], filters,
filter3, filter4);
d = vqrshrun_n_s16(vcombine_s16(tt, tt), 7);
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d), 0);
} else {
memcpy(dst, &src_y[3 * src_stride], w);
}
dst += dst_stride;
y_q4 += y_step_q4;
} while (--y);
}
void vp9_tm_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
int j;
uint16x8_t q0u16, q3u16, q10u16;
int16x8_t q0s16;
uint16x4_t d20u16;
uint8x8_t d0u8, d2u8, d30u8;
d0u8 = vld1_dup_u8(above - 1);
d30u8 = vld1_u8(left);
d2u8 = vld1_u8(above);
q10u16 = vmovl_u8(d30u8);
q3u16 = vsubl_u8(d2u8, d0u8);
d20u16 = vget_low_u16(q10u16);
for (j = 0; j < 2; j++, d20u16 = vget_high_u16(q10u16)) {
q0u16 = vdupq_lane_u16(d20u16, 0);
q0s16 = vaddq_s16(vreinterpretq_s16_u16(q3u16),
vreinterpretq_s16_u16(q0u16));
d0u8 = vqmovun_s16(q0s16);
vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d0u8));
dst += stride;
q0u16 = vdupq_lane_u16(d20u16, 1);
q0s16 = vaddq_s16(vreinterpretq_s16_u16(q3u16),
vreinterpretq_s16_u16(q0u16));
d0u8 = vqmovun_s16(q0s16);
vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d0u8));
dst += stride;
q0u16 = vdupq_lane_u16(d20u16, 2);
q0s16 = vaddq_s16(vreinterpretq_s16_u16(q3u16),
vreinterpretq_s16_u16(q0u16));
d0u8 = vqmovun_s16(q0s16);
vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d0u8));
dst += stride;
q0u16 = vdupq_lane_u16(d20u16, 3);
q0s16 = vaddq_s16(vreinterpretq_s16_u16(q3u16),
vreinterpretq_s16_u16(q0u16));
d0u8 = vqmovun_s16(q0s16);
vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d0u8));
dst += stride;
}
}
void byte2word48_neon(const uint8_t *t, const int pitch, float *pf) {
uint16_t *p = (uint16_t *)pf;
uint8x8_t m0, m1, m2, m3, m4, m5;
m0 = vld1_u8(t);
m1 = vreinterpret_u8_u32(vld1_lane_u32((const uint32_t *)(t + 8), vreinterpret_u32_u8(m1), 0));
m1 = vreinterpret_u8_u32(vld1_lane_u32((const uint32_t *)(t + pitch * 2), vreinterpret_u32_u8(m1), 1));
m2 = vld1_u8(t + pitch * 2 + 4);
t += pitch * 4;
m3 = vld1_u8(t);
m4 = vreinterpret_u8_u32(vld1_lane_u32((const uint32_t *)(t + 8), vreinterpret_u32_u8(m4), 0));
m4 = vreinterpret_u8_u32(vld1_lane_u32((const uint32_t *)(t + pitch * 2), vreinterpret_u32_u8(m4), 1));
m5 = vld1_u8(t + pitch * 2 + 4);
vst1q_u16(p, vmovl_u8(m0));
vst1q_u16(p + 8, vmovl_u8(m1));
vst1q_u16(p + 16, vmovl_u8(m2));
vst1q_u16(p + 24, vmovl_u8(m3));
vst1q_u16(p + 32, vmovl_u8(m4));
vst1q_u16(p + 40, vmovl_u8(m5));
}
void qt_blend_argb32_on_argb32_neon(uchar *destPixels, int dbpl,
const uchar *srcPixels, int sbpl,
int w, int h,
int const_alpha)
{
const uint *src = (const uint *) srcPixels;
uint *dst = (uint *) destPixels;
int16x8_t half = vdupq_n_s16(0x80);
int16x8_t full = vdupq_n_s16(0xff);
if (const_alpha == 256) {
for (int y = 0; y < h; ++y) {
int x = 0;
for (; x < w-3; x += 4) {
int32x4_t src32 = vld1q_s32((int32_t *)&src[x]);
if ((src[x] & src[x+1] & src[x+2] & src[x+3]) >= 0xff000000) {
// all opaque
vst1q_s32((int32_t *)&dst[x], src32);
} else if (src[x] | src[x+1] | src[x+2] | src[x+3]) {
int32x4_t dst32 = vld1q_s32((int32_t *)&dst[x]);
const uint8x16_t src8 = vreinterpretq_u8_s32(src32);
const uint8x16_t dst8 = vreinterpretq_u8_s32(dst32);
const uint8x8_t src8_low = vget_low_u8(src8);
const uint8x8_t dst8_low = vget_low_u8(dst8);
const uint8x8_t src8_high = vget_high_u8(src8);
const uint8x8_t dst8_high = vget_high_u8(dst8);
const int16x8_t src16_low = vreinterpretq_s16_u16(vmovl_u8(src8_low));
const int16x8_t dst16_low = vreinterpretq_s16_u16(vmovl_u8(dst8_low));
const int16x8_t src16_high = vreinterpretq_s16_u16(vmovl_u8(src8_high));
const int16x8_t dst16_high = vreinterpretq_s16_u16(vmovl_u8(dst8_high));
const int16x8_t result16_low = qvsource_over_s16(src16_low, dst16_low, half, full);
const int16x8_t result16_high = qvsource_over_s16(src16_high, dst16_high, half, full);
const int32x2_t result32_low = vreinterpret_s32_s8(vmovn_s16(result16_low));
const int32x2_t result32_high = vreinterpret_s32_s8(vmovn_s16(result16_high));
vst1q_s32((int32_t *)&dst[x], vcombine_s32(result32_low, result32_high));
}
}
for (; x<w; ++x) {
uint s = src[x];
if (s >= 0xff000000)
dst[x] = s;
else if (s != 0)
dst[x] = s + BYTE_MUL(dst[x], qAlpha(~s));
}
dst = (quint32 *)(((uchar *) dst) + dbpl);
src = (const quint32 *)(((const uchar *) src) + sbpl);
}
} else if (const_alpha != 0) {
const_alpha = (const_alpha * 255) >> 8;
int16x8_t const_alpha16 = vdupq_n_s16(const_alpha);
for (int y = 0; y < h; ++y) {
int x = 0;
for (; x < w-3; x += 4) {
if (src[x] | src[x+1] | src[x+2] | src[x+3]) {
int32x4_t src32 = vld1q_s32((int32_t *)&src[x]);
int32x4_t dst32 = vld1q_s32((int32_t *)&dst[x]);
const uint8x16_t src8 = vreinterpretq_u8_s32(src32);
const uint8x16_t dst8 = vreinterpretq_u8_s32(dst32);
const uint8x8_t src8_low = vget_low_u8(src8);
const uint8x8_t dst8_low = vget_low_u8(dst8);
const uint8x8_t src8_high = vget_high_u8(src8);
const uint8x8_t dst8_high = vget_high_u8(dst8);
const int16x8_t src16_low = vreinterpretq_s16_u16(vmovl_u8(src8_low));
const int16x8_t dst16_low = vreinterpretq_s16_u16(vmovl_u8(dst8_low));
const int16x8_t src16_high = vreinterpretq_s16_u16(vmovl_u8(src8_high));
const int16x8_t dst16_high = vreinterpretq_s16_u16(vmovl_u8(dst8_high));
const int16x8_t srcalpha16_low = qvbyte_mul_s16(src16_low, const_alpha16, half);
const int16x8_t srcalpha16_high = qvbyte_mul_s16(src16_high, const_alpha16, half);
const int16x8_t result16_low = qvsource_over_s16(srcalpha16_low, dst16_low, half, full);
const int16x8_t result16_high = qvsource_over_s16(srcalpha16_high, dst16_high, half, full);
const int32x2_t result32_low = vreinterpret_s32_s8(vmovn_s16(result16_low));
const int32x2_t result32_high = vreinterpret_s32_s8(vmovn_s16(result16_high));
vst1q_s32((int32_t *)&dst[x], vcombine_s32(result32_low, result32_high));
}
}
for (; x<w; ++x) {
uint s = src[x];
if (s != 0) {
s = BYTE_MUL(s, const_alpha);
dst[x] = s + BYTE_MUL(dst[x], qAlpha(~s));
}
}
dst = (quint32 *)(((uchar *) dst) + dbpl);
src = (const quint32 *)(((const uchar *) src) + sbpl);
}
}
void vp9_tm_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
int j, k;
uint16x8_t q0u16, q3u16, q8u16, q9u16, q10u16, q11u16;
uint8x16_t q0u8, q1u8, q2u8;
int16x8_t q12s16, q13s16, q14s16, q15s16;
uint16x4_t d6u16;
uint8x8_t d0u8, d1u8, d2u8, d3u8, d26u8;
q0u8 = vld1q_dup_u8(above - 1);
q1u8 = vld1q_u8(above);
q2u8 = vld1q_u8(above + 16);
q8u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q0u8));
q9u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q0u8));
q10u16 = vsubl_u8(vget_low_u8(q2u8), vget_low_u8(q0u8));
q11u16 = vsubl_u8(vget_high_u8(q2u8), vget_high_u8(q0u8));
for (k = 0; k < 4; k++, left += 8) {
d26u8 = vld1_u8(left);
q3u16 = vmovl_u8(d26u8);
d6u16 = vget_low_u16(q3u16);
for (j = 0; j < 2; j++, d6u16 = vget_high_u16(q3u16)) {
q0u16 = vdupq_lane_u16(d6u16, 0);
q12s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q8u16));
q13s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q9u16));
q14s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q10u16));
q15s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q11u16));
d0u8 = vqmovun_s16(q12s16);
d1u8 = vqmovun_s16(q13s16);
d2u8 = vqmovun_s16(q14s16);
d3u8 = vqmovun_s16(q15s16);
q0u8 = vcombine_u8(d0u8, d1u8);
q1u8 = vcombine_u8(d2u8, d3u8);
vst1q_u64((uint64_t *)dst, vreinterpretq_u64_u8(q0u8));
vst1q_u64((uint64_t *)(dst + 16), vreinterpretq_u64_u8(q1u8));
dst += stride;
q0u16 = vdupq_lane_u16(d6u16, 1);
q12s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q8u16));
q13s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q9u16));
q14s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q10u16));
q15s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q11u16));
d0u8 = vqmovun_s16(q12s16);
d1u8 = vqmovun_s16(q13s16);
d2u8 = vqmovun_s16(q14s16);
d3u8 = vqmovun_s16(q15s16);
q0u8 = vcombine_u8(d0u8, d1u8);
q1u8 = vcombine_u8(d2u8, d3u8);
vst1q_u64((uint64_t *)dst, vreinterpretq_u64_u8(q0u8));
vst1q_u64((uint64_t *)(dst + 16), vreinterpretq_u64_u8(q1u8));
dst += stride;
q0u16 = vdupq_lane_u16(d6u16, 2);
q12s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q8u16));
q13s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q9u16));
q14s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q10u16));
q15s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q11u16));
d0u8 = vqmovun_s16(q12s16);
d1u8 = vqmovun_s16(q13s16);
d2u8 = vqmovun_s16(q14s16);
d3u8 = vqmovun_s16(q15s16);
q0u8 = vcombine_u8(d0u8, d1u8);
q1u8 = vcombine_u8(d2u8, d3u8);
vst1q_u64((uint64_t *)dst, vreinterpretq_u64_u8(q0u8));
vst1q_u64((uint64_t *)(dst + 16), vreinterpretq_u64_u8(q1u8));
dst += stride;
q0u16 = vdupq_lane_u16(d6u16, 3);
q12s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q8u16));
q13s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q9u16));
q14s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q10u16));
q15s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q11u16));
d0u8 = vqmovun_s16(q12s16);
d1u8 = vqmovun_s16(q13s16);
d2u8 = vqmovun_s16(q14s16);
d3u8 = vqmovun_s16(q15s16);
q0u8 = vcombine_u8(d0u8, d1u8);
q1u8 = vcombine_u8(d2u8, d3u8);
vst1q_u64((uint64_t *)dst, vreinterpretq_u64_u8(q0u8));
vst1q_u64((uint64_t *)(dst + 16), vreinterpretq_u64_u8(q1u8));
dst += stride;
}
}
}
void yuv422rgb_neon_int(const unsigned char * sourcep, int source_byte_count,
unsigned char * destp)
{
const unsigned char *source_endp;
const unsigned char *vector_endp;
int remainder;
const int16x8_t u_coeff = {0, -22, 113, 0, 0, -22, 113, 0};
const int16x8_t v_coeff = {90, -46, 0, 0, 90, -46, 0, 0};
const uint8x8_t zeroalpha = {0x0, 0x0, 0x0, 0xFF, 0x0, 0x0, 0x0, 0xFF};
const int16x8_t uvbias = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
int16x8_t mp0_rgba; /* macropixel 0's resulting RGBA RGBA pixels */
int16x8_t mp1_rgba; /* macropixel 1's resulting RGBA RGBA pixels */
uint8x8_t rawpixels; /* source pixels as {[YUYV]0 [YUYV]1} */
uint8x8_t rgba0, rgba1; /* rgba values as bytes */
uint8x16_t bothrgba;
uint8_t * destinationp; /* pointer into output buffer destp */
int16x8_t widerpixels; /* rawpixels promoted to shorts per component */
const uint8x8_t yselect = {0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00};
/* we're working with things in 4-byte macropixels */
remainder = source_byte_count % 4;
source_endp = sourcep + source_byte_count;
vector_endp = source_endp - remainder;
destinationp = (uint8_t *)destp;
while (sourcep < vector_endp)
{
/* pull YUYV from 2 four byte macropixels starting at sourcep. */
/* we'll increment sourcep as we go to save the array dereference */
/* and separate increment instruction at the end of the loop */
/* load rawpixels with {[YUYV]0 [YUYV]1 } with byte components */
rawpixels = vld1_u8(sourcep);
sourcep += sizeof(rawpixels);
widerpixels = vreinterpretq_s16_u16(vmovl_u8(rawpixels));
/* ---------- process macropixel 0 --------------- */
/* take macropixel zero ([YUYV]0) from rawpixels and */
/* compute the two RGBA pixels that come from it. store */
/* those two pixels in mp0_rgba */
{
int16x8_t wider_yalpha;
int16x8_t u_vec, v_vec, uv_vec;
uint8x8_t narrow_yalpha;
uint8x8_t y0_vec, y1_vec;
int16x4_t yuyv;
/* narrow_yalpha is drawn from [YUYV]0 and formed into */
/* {Y0, Y0, Y0, alpha, Y1, Y1, Y1, alpha} */
/* this would have been a nice place for vtbx1_u8, but i */
/* can't get it to work. so i'll have to use vbsl_u8 instead. */
y0_vec = vdup_lane_u8(rawpixels, MP0_Y0);
y1_vec = vdup_lane_u8(rawpixels, MP0_Y1);
narrow_yalpha = vbsl_u8(yselect, y0_vec, y1_vec);
/* store ALPHA in elements 3 and 7 (after the RGB components) */
narrow_yalpha = vset_lane_u8(ALPHA, narrow_yalpha, 3);
narrow_yalpha = vset_lane_u8(ALPHA, narrow_yalpha, 7);
/* use vmovl_u8 to go from being unsigned 8-bit to */
/* unsigned 16-bit, the use vreinterpretq_s16_u16 to */
/* change interpretation from unsigned 16-bit to signed */
/* 16-bit. */
wider_yalpha = vreinterpretq_s16_u16(vmovl_u8(narrow_yalpha));
yuyv = vget_low_s16(widerpixels);
/* form a vector of the U component from MP0 */
u_vec = vdupq_lane_s16(yuyv, MP0_U);
/* subtract uvbias from u_vec */
u_vec = vsubq_s16(u_vec, uvbias);
/* form a vector of the V component from MP0 */
v_vec = vdupq_lane_s16(yuyv, MP0_V);
/* subtract uvbias from v_vec */
v_vec = vsubq_s16(v_vec, uvbias);
/* Multiply eight 16-bit values in u_vec by eight 16-bit */
/* values in u_coeff and store the results in u_vec. */
u_vec = vmulq_s16(u_vec, u_coeff);
/* likewise multiply eight 16-bit values in v_vec by */
/* v_coeff and store the results in v_vec */
v_vec = vmulq_s16(v_vec, v_coeff);
/* form uv_vec as the sum of u_vec & v_vec, then shift 6 places */
/* (dividing by 64) */
uv_vec = vaddq_s16(u_vec, v_vec);
uv_vec = vshrq_n_s16(uv_vec, 6);
/* now mp0_rgba = y_vec + u_vec + v_vec */
mp0_rgba = vaddq_s16(wider_yalpha, uv_vec);
}
/* ---------- process macropixel 1 --------------- */
/* take macropixel one ([YUYV]1) from rawpixels and */
/* compute the two RGBA pixels that come from it. store */
/* those two pixels in mp1_rgba */
{
int16x8_t wider_yalpha;
int16x8_t u_vec, v_vec, uv_vec;
uint8x8_t narrow_yalpha;
uint8x8_t y0_vec, y1_vec;
int16x4_t yuyv;
/* narrow_yalpha is drawn from [YUYV]1 and formed into */
/* {Y0, Y0, Y0, alpha, Y1, Y1, Y1, alpha} */
/* this would have been a nice place for vtbx1_u8, but i */
/* can't get it to work. so i'll have to use vbsl_u8 instead. */
y0_vec = vdup_lane_u8(rawpixels, MP1_Y0);
y1_vec = vdup_lane_u8(rawpixels, MP1_Y1);
narrow_yalpha = vbsl_u8(yselect, y0_vec, y1_vec);
narrow_yalpha = vset_lane_u8(ALPHA, narrow_yalpha, 3);
narrow_yalpha = vset_lane_u8(ALPHA, narrow_yalpha, 7);
/* use vmovl_u8 to go from being unsigned 8-bit to */
/* unsigned 16-bit, the use vreinterpretq_s16_u16 to */
wider_yalpha = vreinterpretq_s16_u16(vmovl_u8(narrow_yalpha));
yuyv = vget_high_s16(widerpixels);
u_vec = vdupq_lane_s16(yuyv, 1);
u_vec = vsubq_s16(u_vec, uvbias);
v_vec = vdupq_lane_s16(yuyv, 3);
v_vec = vsubq_s16(v_vec, uvbias);
/* Multiply eight 16-bit values in u_vec by eight 16-bit */
/* values in u_coeff and store the results in u_vec. */
u_vec = vmulq_s16(u_vec, u_coeff);
/* likewise multiply eight 16-bit values in v_vec by */
/* v_coeff and store the results in v_vec */
v_vec = vmulq_s16(v_vec, v_coeff);
/* form uv_vec as the sum of u_vec & v_vec, then shift 6 places */
/* (dividing by 64) */
uv_vec = vaddq_s16(u_vec, v_vec);
uv_vec = vshrq_n_s16(uv_vec, 6);
/* now mp1_rgba = y_vec + u_vec + v_vec */
mp1_rgba = vaddq_s16(wider_yalpha, uv_vec);
}
/* turn mp0_rgba from a vector of shorts to a vector of */
/* unsigned unsigned chars. this will saturate: clipping */
/* the values between 0 and 255. */
rgba0 = vqmovun_s16(mp0_rgba);
rgba1 = vqmovun_s16(mp1_rgba);
/* make it faster to copy these back out of vector registers into */
/* memory by combining rgba0 and rgba1 into the larger bothrgba. */
/* then store that back into memory at destinationp. */
bothrgba = vcombine_u8(rgba0, rgba1);
vst1q_u8(destinationp, bothrgba);
destinationp += 16;
}
}
x255_16x8 = vdupq_n_u16(0xff); # ifdef COLMUL uint16x4_t x255_16x4; x255_16x4 = vget_low_u16(x255_16x8); uint16x4_t c1_16x4; # ifdef COLSAME uint16x4_t c1_val3_16x4; uint16x8_t c1_16x8; uint16x8_t c1_val3_16x8; uint32x2_t c1_32x2; uint8x8_t c1_8x8; uint8x8_t c1_val3_8x8; c1_32x2 = vset_lane_u32(c1, c1_32x2, 0); c1_8x8 = vreinterpret_u8_u32(c1_32x2); c1_16x8 = vmovl_u8(c1_8x8); c1_16x4 = vget_low_u16(c1_16x8); # else uint16x4_t c2_16x4; uint16x4_t c2_local_16x4; uint16x4_t cv_16x4; uint16x8_t c1_c2_16x8; uint16x8_t c1_val1_16x8; uint16x8_t c2_val3_16x8; uint16x8_t cv_rv_16x8; uint32x2_t c1_c2_32x2; uint8x8_t c1_c2_8x8; uint8x8_t val3_8x8; uint16x8_t val3_16x8; c1_c2_32x2 = vset_lane_u32(c1, c1_c2_32x2, 0);
XnStatus Link12BitS2DParser::Unpack12to16(const XnUInt8* pcInput,XnUInt8* pDest, const XnUInt32 nInputSize, XnUInt32* pnActualRead, XnUInt32* pnActualWritten)
{
const XnUInt8* pOrigInput = (XnUInt8*)pcInput;
XnUInt32 nElements = nInputSize / XN_INPUT_ELEMENT_SIZE; // floored
//XnUInt32 nNeededOutput = nElements * XN_OUTPUT_ELEMENT_SIZE;
*pnActualRead = 0;
XnUInt16 *pnOutput = (XnUInt16*)pDest;
XnUInt16 shift[16];
#ifdef XN_NEON
XnUInt16 depth[16];
uint8x8x3_t inD3;
uint8x8_t rshft4D, lshft4D;
uint16x8_t rshft4Q, lshft4Q;
uint16x8_t depthQ;
uint16x8x2_t shiftQ2;
#endif
// Convert the 11bit packed data into 16bit shorts
for (XnUInt32 nElem = 0; nElem < nElements; ++nElem)
{
#ifndef XN_NEON
// input: 0, 1,2,3, 4,5,6, 7,8,9, 10,11,12, 13,14,15, 16,17,18, 19,20,21, 22,23
// -,---,-,-,---,-,-,---,-,-,---,--,--,---,--,--,---,--,--,---,--,--,---,--
// bits: 8,4,4,8,8,4,4,8,8,4,4,8,8,4,4, 8, 8,4,4, 8, 8,4,4, 8, 8,4,4, 8, 8,4,4, 8
// ---,---,---,---,---,---,---,----,----,----,----,----,----,----,----,----
// output: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
shift[0] = (XN_TAKE_BITS(pcInput[0],8,0) << 4) | XN_TAKE_BITS(pcInput[1],4,4);
shift[1] = (XN_TAKE_BITS(pcInput[1],4,0) << 8) | XN_TAKE_BITS(pcInput[2],8,0);
shift[2] = (XN_TAKE_BITS(pcInput[3],8,0) << 4) | XN_TAKE_BITS(pcInput[4],4,4);
shift[3] = (XN_TAKE_BITS(pcInput[4],4,0) << 8) | XN_TAKE_BITS(pcInput[5],8,0);
shift[4] = (XN_TAKE_BITS(pcInput[6],8,0) << 4) | XN_TAKE_BITS(pcInput[7],4,4);
shift[5] = (XN_TAKE_BITS(pcInput[7],4,0) << 8) | XN_TAKE_BITS(pcInput[8],8,0);
shift[6] = (XN_TAKE_BITS(pcInput[9],8,0) << 4) | XN_TAKE_BITS(pcInput[10],4,4);
shift[7] = (XN_TAKE_BITS(pcInput[10],4,0) << 8) | XN_TAKE_BITS(pcInput[11],8,0);
shift[8] = (XN_TAKE_BITS(pcInput[12],8,0) << 4) | XN_TAKE_BITS(pcInput[13],4,4);
shift[9] = (XN_TAKE_BITS(pcInput[13],4,0) << 8) | XN_TAKE_BITS(pcInput[14],8,0);
shift[10] = (XN_TAKE_BITS(pcInput[15],8,0) << 4) | XN_TAKE_BITS(pcInput[16],4,4);
shift[11] = (XN_TAKE_BITS(pcInput[16],4,0) << 8) | XN_TAKE_BITS(pcInput[17],8,0);
shift[12] = (XN_TAKE_BITS(pcInput[18],8,0) << 4) | XN_TAKE_BITS(pcInput[19],4,4);
shift[13] = (XN_TAKE_BITS(pcInput[19],4,0) << 8) | XN_TAKE_BITS(pcInput[20],8,0);
shift[14] = (XN_TAKE_BITS(pcInput[21],8,0) << 4) | XN_TAKE_BITS(pcInput[22],4,4);
shift[15] = (XN_TAKE_BITS(pcInput[22],4,0) << 8) | XN_TAKE_BITS(pcInput[23],8,0);
pnOutput[0] = m_pShiftToDepth[(shift[0])];
pnOutput[1] = m_pShiftToDepth[(shift[1])];
pnOutput[2] = m_pShiftToDepth[(shift[2])];
pnOutput[3] = m_pShiftToDepth[(shift[3])];
pnOutput[4] = m_pShiftToDepth[(shift[4])];
pnOutput[5] = m_pShiftToDepth[(shift[5])];
pnOutput[6] = m_pShiftToDepth[(shift[6])];
pnOutput[7] = m_pShiftToDepth[(shift[7])];
pnOutput[8] = m_pShiftToDepth[(shift[8])];
pnOutput[9] = m_pShiftToDepth[(shift[9])];
pnOutput[10] = m_pShiftToDepth[(shift[10])];
pnOutput[11] = m_pShiftToDepth[(shift[11])];
pnOutput[12] = m_pShiftToDepth[(shift[12])];
pnOutput[13] = m_pShiftToDepth[(shift[13])];
pnOutput[14] = m_pShiftToDepth[(shift[14])];
pnOutput[15] = m_pShiftToDepth[(shift[15])];
#else
// input: 0, 1,2 (X8)
// -,---,-
// bits: 8,4,4,8 (X8)
// ---,---
// output: 0, 1 (X8)
// Split 24 bytes into 3 vectors (64 bit each)
inD3 = vld3_u8(pcInput);
// rshft4D0 contains 4 MSB of second vector (placed at offset 0)
rshft4D = vshr_n_u8(inD3.val[1], 4);
// lshft4D0 contains 4 LSB of second vector (placed at offset 4)
lshft4D = vshl_n_u8(inD3.val[1], 4);
// Expand 64 bit vectors to 128 bit (8 values of 16 bits)
shiftQ2.val[0] = vmovl_u8(inD3.val[0]);
shiftQ2.val[1] = vmovl_u8(inD3.val[2]);
rshft4Q = vmovl_u8(rshft4D);
lshft4Q = vmovl_u8(lshft4D);
// Even indexed shift = 8 bits from first vector + 4 MSB bits of second vector
shiftQ2.val[0] = vshlq_n_u16(shiftQ2.val[0], 4);
shiftQ2.val[0] = vorrq_u16(shiftQ2.val[0], rshft4Q);
// Odd indexed shift = 4 LSB bits of second vector + 8 bits from third vector
lshft4Q = vshlq_n_u16(lshft4Q, 4);
shiftQ2.val[1] = vorrq_u16(shiftQ2.val[1], lshft4Q);
// Interleave shift values to a single vector
vst2q_u16(shift, shiftQ2);
depth[0] = m_pShiftToDepth[(shift[0])];
depth[1] = m_pShiftToDepth[(shift[1])];
depth[2] = m_pShiftToDepth[(shift[2])];
depth[3] = m_pShiftToDepth[(shift[3])];
depth[4] = m_pShiftToDepth[(shift[4])];
depth[5] = m_pShiftToDepth[(shift[5])];
depth[6] = m_pShiftToDepth[(shift[6])];
depth[7] = m_pShiftToDepth[(shift[7])];
// Load
depthQ = vld1q_u16(depth);
//Store
vst1q_u16(pnOutput, depthQ);
depth[8] = m_pShiftToDepth[(shift[8])];
depth[9] = m_pShiftToDepth[(shift[9])];
depth[10] = m_pShiftToDepth[(shift[10])];
depth[11] = m_pShiftToDepth[(shift[11])];
depth[12] = m_pShiftToDepth[(shift[12])];
depth[13] = m_pShiftToDepth[(shift[13])];
depth[14] = m_pShiftToDepth[(shift[14])];
depth[15] = m_pShiftToDepth[(shift[15])];
// Load
depthQ = vld1q_u16(depth + 8);
// Store
vst1q_u16(pnOutput + 8, depthQ);
#endif
pcInput += XN_INPUT_ELEMENT_SIZE;
pnOutput += 16;
}
*pnActualRead = (XnUInt32)(pcInput - pOrigInput); // total bytes
*pnActualWritten = (XnUInt32)((XnUInt8*)pnOutput - pDest);
return XN_STATUS_OK;
}
bool decode_yuv_neon(unsigned char* out, unsigned char const* y, unsigned char const* uv, int width, int height, unsigned char fill_alpha=0xff)
{
// pre-condition : width, height must be even
if (0!=(width&1) || width<2 || 0!=(height&1) || height<2 || !out || !y || !uv)
return false;
// in & out pointers
unsigned char* dst = out;
// constants
int const stride = width*trait::bytes_per_pixel;
int const itHeight = height>>1;
int const itWidth = width>>3;
uint8x8_t const Yshift = vdup_n_u8(16);
int16x8_t const half = vdupq_n_u16(128);
int32x4_t const rounding = vdupq_n_s32(128);
// tmp variable
uint16x8_t t;
// pixel block to temporary store 8 pixels
typename trait::PixelBlock pblock = trait::init_pixelblock(fill_alpha);
for (int j=0; j<itHeight; ++j, y+=width, dst+=stride) {
for (int i=0; i<itWidth; ++i, y+=8, uv+=8, dst+=(8*trait::bytes_per_pixel)) {
t = vmovl_u8(vqsub_u8(vld1_u8(y), Yshift));
int32x4_t const Y00 = vmulq_n_u32(vmovl_u16(vget_low_u16(t)), 298);
int32x4_t const Y01 = vmulq_n_u32(vmovl_u16(vget_high_u16(t)), 298);
t = vmovl_u8(vqsub_u8(vld1_u8(y+width), Yshift));
int32x4_t const Y10 = vmulq_n_u32(vmovl_u16(vget_low_u16(t)), 298);
int32x4_t const Y11 = vmulq_n_u32(vmovl_u16(vget_high_u16(t)), 298);
// trait::loadvu pack 4 sets of uv into a uint8x8_t, layout : { v0,u0, v1,u1, v2,u2, v3,u3 }
t = vsubq_s16((int16x8_t)vmovl_u8(trait::loadvu(uv)), half);
// UV.val[0] : v0, v1, v2, v3
// UV.val[1] : u0, u1, u2, u3
int16x4x2_t const UV = vuzp_s16(vget_low_s16(t), vget_high_s16(t));
// tR : 128+409V
// tG : 128-100U-208V
// tB : 128+516U
int32x4_t const tR = vmlal_n_s16(rounding, UV.val[0], 409);
int32x4_t const tG = vmlal_n_s16(vmlal_n_s16(rounding, UV.val[0], -208), UV.val[1], -100);
int32x4_t const tB = vmlal_n_s16(rounding, UV.val[1], 516);
int32x4x2_t const R = vzipq_s32(tR, tR); // [tR0, tR0, tR1, tR1] [ tR2, tR2, tR3, tR3]
int32x4x2_t const G = vzipq_s32(tG, tG); // [tG0, tG0, tG1, tG1] [ tG2, tG2, tG3, tG3]
int32x4x2_t const B = vzipq_s32(tB, tB); // [tB0, tB0, tB1, tB1] [ tB2, tB2, tB3, tB3]
// upper 8 pixels
trait::store_pixel_block(dst, pblock,
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(R.val[0], Y00)), vqmovun_s32(vaddq_s32(R.val[1], Y01))), 8),
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(G.val[0], Y00)), vqmovun_s32(vaddq_s32(G.val[1], Y01))), 8),
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(B.val[0], Y00)), vqmovun_s32(vaddq_s32(B.val[1], Y01))), 8));
// lower 8 pixels
trait::store_pixel_block(dst+stride, pblock,
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(R.val[0], Y10)), vqmovun_s32(vaddq_s32(R.val[1], Y11))), 8),
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(G.val[0], Y10)), vqmovun_s32(vaddq_s32(G.val[1], Y11))), 8),
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(B.val[0], Y10)), vqmovun_s32(vaddq_s32(B.val[1], Y11))), 8));
}
}
return true;
}
//
// box blur a square array of pixels (power of 2, actually)
// if we insist on powers of 2, we don't need to special case some end-of-row/col conditions
// to a specific blur width
//
// also, we're using NEON to vectorize our arithmetic.
// we need to do a division along the way, but NEON doesn't support integer division.
// so rather than divide by, say "w", we multiply by magic(w).
// magic(w) is chosen so that the result of multiplying by it will be the same as
// dividing by w, except that the result will be in the high half of the result.
// yes, dorothy... this is what compilers do, too...
void NEONboxBlur(pixel *src, pixel *dest, unsigned int size, unsigned int blurRad) {
unsigned int wid = 2 * blurRad + 1;
// because NEON doesn't have integer division, we use "magic constants" that will give
// use the result of division by multiplication -- the upper half of the result will be
// (more or less) the result of the division.
// for this, we need to compute the magic numbers corresponding to a given divisor
struct magicu_info minfo = compute_unsigned_magic_info(wid, 16);
int16x8_t preshift = vdupq_n_s16(-minfo.pre_shift); // negative means shift right
int32x4_t postshift = vdupq_n_s32(-(minfo.post_shift+16)); // negative means shift right
uint16x4_t magic = vdup_n_u16(minfo.multiplier);
// fprintf(stderr,"width %5d, preshift %d, postshift %d + 16, increment %d, magic %d\n", wid,
// minfo.pre_shift, minfo.post_shift, minfo.increment, minfo.multiplier);
// if (minfo.pre_shift > 0) fprintf(stderr,"hey, not an odd number!\n");
int i, j, k, ch;
for (i = 0 ; i < size ; i+=8) {
// first, initialize the sum so that we can loop from 0 to size-1
// we'll initialize boxsum for index -1, so that we can move into 0 as part of our loop
uint16x8x4_t boxsum;
uint8x8x4_t firstpixel = vld4_u8((uint8_t *)(src + 0 * size + i));
for (ch = 0 ; ch < 4 ; ch++) {
// boxsum[ch] = blurRad * srcpixel[ch]
boxsum.val[ch] = vmulq_n_u16(vmovl_u8(firstpixel.val[ch]),(blurRad+1)+1);
}
for ( k = 1 ; k < blurRad ; k++) {
uint8x8x4_t srcpixel = vld4_u8((uint8_t *)(src + k * size + i));
for (ch = 0 ; ch < 4 ; ch++ ) {
boxsum.val[ch] = vaddw_u8(boxsum.val[ch], srcpixel.val[ch]);
}
}
int right = blurRad-1;
int left = -blurRad-1;
if (minfo.increment) {
for ( k = 0 ; k < size ; k++) {
// move to next pixel
unsigned int l = (left < 0)?0:left; // take off the old left
left++;
right++;
unsigned int r = (right < size)?right:(size-1); // but add the new right
uint8x8x4_t addpixel = vld4_u8((uint8_t *)(src + r * size + i));
uint8x8x4_t subpixel = vld4_u8((uint8_t *)(src + l * size + i));
for (ch = 0 ; ch < 4 ; ch++ ) {
// boxsum[ch] += addpixel[ch] - subpixel[ch];
boxsum.val[ch] = vsubw_u8(vaddw_u8(boxsum.val[ch], addpixel.val[ch]), subpixel.val[ch]);
}
uint8x8x4_t destpixel;
for (ch = 0 ; ch < 4 ; ch++ ) { // compute: destpixel = boxsum / wid
// since 16bit multiplication leads to 32bit results, we need to
// split our task into two chunks, for the hi and low half of our vector
// (because otherwise, it won't all fit into 128 bits)
// this is the meat of the magic division algorithm (see the include file...)
uint16x8_t bsum_preshifted = vshlq_u16(boxsum.val[ch],preshift);
// multiply by the magic number
uint32x4_t res_hi = vmull_u16(vget_high_u16(bsum_preshifted), magic);
res_hi = vaddw_u16(res_hi, magic);
// take the high half and post-shift
uint16x4_t q_hi = vmovn_u32(vshlq_u32(res_hi, postshift));
// pre-shift and multiply by the magic number
uint32x4_t res_lo = vmull_u16(vget_low_u16(bsum_preshifted), magic);
res_lo = vaddw_u16(res_lo, magic);
// take the high half and post-shift
uint16x4_t q_lo = vmovn_u32(vshlq_u32(res_lo, postshift));
destpixel.val[ch] = vqmovn_u16(vcombine_u16(q_lo, q_hi));
}
pixel block[8];
vst4_u8((uint8_t *)&block, destpixel);
for (j = 0 ; j < 8 ; j++ ) {
dest[(i + j)*size + k] = block[j];
}
// vst4_u8((uint8_t *)(dest + k * size + i), destpixel);
}
} else {
for ( k = 0 ; k < size ; k++) {
// move to next pixel
unsigned int l = (left < 0)?0:left; // take off the old left
left++;
right++;
unsigned int r = (right < size)?right:(size-1); // but add the new right
uint8x8x4_t addpixel = vld4_u8((uint8_t *)(src + r * size + i));
uint8x8x4_t subpixel = vld4_u8((uint8_t *)(src + l * size + i));
for (ch = 0 ; ch < 4 ; ch++ ) {
// boxsum[ch] += addpixel[ch] - subpixel[ch];
boxsum.val[ch] = vsubw_u8(vaddw_u8(boxsum.val[ch], addpixel.val[ch]), subpixel.val[ch]);
}
uint8x8x4_t destpixel;
for (ch = 0 ; ch < 4 ; ch++ ) { // compute: destpixel = boxsum / wid
// since 16bit multiplication leads to 32bit results, we need to
// split our task into two chunks, for the hi and low half of our vector
// (because otherwise, it won't all fit into 128 bits)
// this is the meat of the magic division algorithm (see the include file...)
uint16x8_t bsum_preshifted = vshlq_u16(boxsum.val[ch],preshift);
// multiply by the magic number
// take the high half and post-shift
uint32x4_t res_hi = vmull_u16(vget_high_u16(bsum_preshifted), magic);
uint16x4_t q_hi = vmovn_u32(vshlq_u32(res_hi, postshift));
// multiply by the magic number
// take the high half and post-shift
uint32x4_t res_lo = vmull_u16(vget_low_u16(bsum_preshifted), magic);
uint16x4_t q_lo = vmovn_u32(vshlq_u32(res_lo, postshift));
destpixel.val[ch] = vqmovn_u16(vcombine_u16(q_lo, q_hi));
}
pixel block[8];
vst4_u8((uint8_t *)&block, destpixel);
for (j = 0 ; j < 8 ; j++ ) {
dest[(i + j)*size + k] = block[j];
}
// vst4_u8((uint8_t *)(dest + k * size + i), destpixel);
}
}
}
}
void vp9_tm_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
int j, k;
uint16x8_t q0u16, q2u16, q3u16, q8u16, q10u16;
uint8x16_t q0u8, q1u8;
int16x8_t q0s16, q1s16, q8s16, q11s16;
uint16x4_t d20u16;
uint8x8_t d2u8, d3u8, d18u8, d22u8, d23u8;
q0u8 = vld1q_dup_u8(above - 1);
q1u8 = vld1q_u8(above);
q2u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q0u8));
q3u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q0u8));
for (k = 0; k < 2; k++, left += 8) {
d18u8 = vld1_u8(left);
q10u16 = vmovl_u8(d18u8);
d20u16 = vget_low_u16(q10u16);
for (j = 0; j < 2; j++, d20u16 = vget_high_u16(q10u16)) {
q0u16 = vdupq_lane_u16(d20u16, 0);
q8u16 = vdupq_lane_u16(d20u16, 1);
q1s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q2u16));
q0s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q3u16));
q11s16 = vaddq_s16(vreinterpretq_s16_u16(q8u16),
vreinterpretq_s16_u16(q2u16));
q8s16 = vaddq_s16(vreinterpretq_s16_u16(q8u16),
vreinterpretq_s16_u16(q3u16));
d2u8 = vqmovun_s16(q1s16);
d3u8 = vqmovun_s16(q0s16);
d22u8 = vqmovun_s16(q11s16);
d23u8 = vqmovun_s16(q8s16);
vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d2u8));
vst1_u64((uint64_t *)(dst + 8), vreinterpret_u64_u8(d3u8));
dst += stride;
vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d22u8));
vst1_u64((uint64_t *)(dst + 8), vreinterpret_u64_u8(d23u8));
dst += stride;
q0u16 = vdupq_lane_u16(d20u16, 2);
q8u16 = vdupq_lane_u16(d20u16, 3);
q1s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q2u16));
q0s16 = vaddq_s16(vreinterpretq_s16_u16(q0u16),
vreinterpretq_s16_u16(q3u16));
q11s16 = vaddq_s16(vreinterpretq_s16_u16(q8u16),
vreinterpretq_s16_u16(q2u16));
q8s16 = vaddq_s16(vreinterpretq_s16_u16(q8u16),
vreinterpretq_s16_u16(q3u16));
d2u8 = vqmovun_s16(q1s16);
d3u8 = vqmovun_s16(q0s16);
d22u8 = vqmovun_s16(q11s16);
d23u8 = vqmovun_s16(q8s16);
vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d2u8));
vst1_u64((uint64_t *)(dst + 8), vreinterpret_u64_u8(d3u8));
dst += stride;
vst1_u64((uint64_t *)dst, vreinterpret_u64_u8(d22u8));
vst1_u64((uint64_t *)(dst + 8), vreinterpret_u64_u8(d23u8));
dst += stride;
}
}
}
static INLINE void scaledconvolve_horiz_w4(
const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
const ptrdiff_t dst_stride, const InterpKernel *const x_filters,
const int x0_q4, const int x_step_q4, const int w, const int h) {
DECLARE_ALIGNED(16, uint8_t, temp[4 * 4]);
int x, y, z;
src -= SUBPEL_TAPS / 2 - 1;
y = h;
do {
int x_q4 = x0_q4;
x = 0;
do {
// process 4 src_x steps
for (z = 0; z < 4; ++z) {
const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
if (x_q4 & SUBPEL_MASK) {
const int16x8_t filters = vld1q_s16(x_filters[x_q4 & SUBPEL_MASK]);
const int16x4_t filter3 = vdup_lane_s16(vget_low_s16(filters), 3);
const int16x4_t filter4 = vdup_lane_s16(vget_high_s16(filters), 0);
uint8x8_t s[8], d;
int16x8_t ss[4];
int16x4_t t[8], tt;
load_u8_8x4(src_x, src_stride, &s[0], &s[1], &s[2], &s[3]);
transpose_u8_8x4(&s[0], &s[1], &s[2], &s[3]);
ss[0] = vreinterpretq_s16_u16(vmovl_u8(s[0]));
ss[1] = vreinterpretq_s16_u16(vmovl_u8(s[1]));
ss[2] = vreinterpretq_s16_u16(vmovl_u8(s[2]));
ss[3] = vreinterpretq_s16_u16(vmovl_u8(s[3]));
t[0] = vget_low_s16(ss[0]);
t[1] = vget_low_s16(ss[1]);
t[2] = vget_low_s16(ss[2]);
t[3] = vget_low_s16(ss[3]);
t[4] = vget_high_s16(ss[0]);
t[5] = vget_high_s16(ss[1]);
t[6] = vget_high_s16(ss[2]);
t[7] = vget_high_s16(ss[3]);
tt = convolve8_4(t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7],
filters, filter3, filter4);
d = vqrshrun_n_s16(vcombine_s16(tt, tt), 7);
vst1_lane_u32((uint32_t *)&temp[4 * z], vreinterpret_u32_u8(d), 0);
} else {
int i;
for (i = 0; i < 4; ++i) {
temp[z * 4 + i] = src_x[i * src_stride + 3];
}
}
x_q4 += x_step_q4;
}
// transpose the 4x4 filters values back to dst
{
const uint8x8x4_t d4 = vld4_u8(temp);
vst1_lane_u32((uint32_t *)&dst[x + 0 * dst_stride],
vreinterpret_u32_u8(d4.val[0]), 0);
vst1_lane_u32((uint32_t *)&dst[x + 1 * dst_stride],
vreinterpret_u32_u8(d4.val[1]), 0);
vst1_lane_u32((uint32_t *)&dst[x + 2 * dst_stride],
vreinterpret_u32_u8(d4.val[2]), 0);
vst1_lane_u32((uint32_t *)&dst[x + 3 * dst_stride],
vreinterpret_u32_u8(d4.val[3]), 0);
}
x += 4;
} while (x < w);
src += src_stride * 4;
dst += dst_stride * 4;
y -= 4;
} while (y > 0);
}
int neon_new(DATA32* _p0, DATA32* _p1, DATA32* _p2, DATA32* _p3, DATA32* _ax, DATA32 _ay, DATA32* result, int len) {
int ay = _ay;
int i;
DATA32* pbuf = result;
uint16x4_t ay_16x4;
uint16x4_t p0_16x4;
uint16x4_t p2_16x4;
uint16x8_t ax_16x8;
uint16x8_t p0_p2_16x8;
uint16x8_t p1_p3_16x8;
uint16x8_t x255_16x8;
uint32x2_t p0_p2_32x2;
uint32x2_t p1_p3_32x2;
uint32x2_t res_32x2;
uint8x8_t p0_p2_8x8;
uint8x8_t p1_p3_8x8;
uint8x8_t p2_8x8;
uint16x4_t temp_16x4;
ay_16x4 = vdup_n_u16(ay);
x255_16x8 = vdupq_n_u16(0xff);
for(i = 0; i < len; i++) {
DATA32 p0 = *_p0++;
DATA32 p1 = *_p1++;
DATA32 p2 = *_p2++;
DATA32 p3 = *_p3++;
int ax = *_ax++;
if (p0 | p1 | p2 | p3)
{
ax_16x8 = vdupq_n_u16(ax);
p0_p2_32x2 = vset_lane_u32(p0, p0_p2_32x2, 0);
p0_p2_32x2 = vset_lane_u32(p2, p0_p2_32x2, 1);
p1_p3_32x2 = vset_lane_u32(p1, p1_p3_32x2, 0);
p1_p3_32x2 = vset_lane_u32(p3, p1_p3_32x2, 1);
p0_p2_8x8 = vreinterpret_u8_u32(p0_p2_32x2);
p1_p3_8x8 = vreinterpret_u8_u32(p1_p3_32x2);
p1_p3_16x8 = vmovl_u8(p1_p3_8x8);
p0_p2_16x8 = vmovl_u8(p0_p2_8x8);
p1_p3_16x8 = vsubq_u16(p1_p3_16x8, p0_p2_16x8);
p1_p3_16x8 = vmulq_u16(p1_p3_16x8, ax_16x8);
p1_p3_16x8 = vshrq_n_u16(p1_p3_16x8, 8);
p1_p3_16x8 = vaddq_u16(p1_p3_16x8, p0_p2_16x8);
p1_p3_16x8 = vandq_u16(p1_p3_16x8, x255_16x8);
p0_16x4 = vget_low_u16(p1_p3_16x8);
p2_16x4 = vget_high_u16(p1_p3_16x8);
p2_16x4 = vsub_u16(p2_16x4, p0_16x4);
p2_16x4 = vmul_u16(p2_16x4, ay_16x4);
p2_16x4 = vshr_n_u16(p2_16x4, 8);
p2_16x4 = vadd_u16(p2_16x4, p0_16x4);
p1_p3_16x8 = vcombine_u16(temp_16x4, p2_16x4);
p2_8x8 = vmovn_u16(p1_p3_16x8);
res_32x2 = vreinterpret_u32_u8(p2_8x8);
vst1_lane_u32(pbuf++, res_32x2, 1);
}
else
*pbuf++ = p0;
}
return 0;
}
void ne10_img_hresize_4channels_linear_neon (const unsigned char** src, int** dst, int count,
const int* xofs, const short* alpha,
int swidth, int dwidth, int cn, int xmin, int xmax)
{
int dx, k;
int dx0 = 0;
int16x4x2_t alpha_vec;
uint8x8_t dS0_vec, dS1_vec;
int16x8_t qS0_vec, qS1_vec;
int16x4_t dS0_0123, dS0_4567, dS1_0123, dS1_4567;
int32x4_t qT0_vec, qT1_vec;
int16x4_t dCoeff;
dCoeff = vdup_n_s16 (INTER_RESIZE_COEF_SCALE);
for (k = 0; k <= count - 2; k++)
{
const unsigned char *S0 = src[k], *S1 = src[k + 1];
int *D0 = dst[k], *D1 = dst[k + 1];
for (dx = dx0; dx < xmax; dx += 4)
{
int sx = xofs[dx];
alpha_vec = vld2_s16 (&alpha[dx * 2]);
dS0_vec = vld1_u8 (&S0[sx]);
dS1_vec = vld1_u8 (&S1[sx]);
qS0_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS0_vec));
qS1_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS1_vec));
dS0_0123 = vget_low_s16 (qS0_vec);
dS0_4567 = vget_high_s16 (qS0_vec);
dS1_0123 = vget_low_s16 (qS1_vec);
dS1_4567 = vget_high_s16 (qS1_vec);
qT0_vec = vmull_s16 (dS0_0123, alpha_vec.val[0]);
qT1_vec = vmull_s16 (dS1_0123, alpha_vec.val[0]);
qT0_vec = vmlal_s16 (qT0_vec, dS0_4567, alpha_vec.val[1]);
qT1_vec = vmlal_s16 (qT1_vec, dS1_4567, alpha_vec.val[1]);
vst1q_s32 (&D0[dx], qT0_vec);
vst1q_s32 (&D1[dx], qT1_vec);
}
for (; dx < dwidth; dx += 4)
{
int sx = xofs[dx];
dS0_vec = vld1_u8 (&S0[sx]);
dS1_vec = vld1_u8 (&S1[sx]);
qS0_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS0_vec));
qS1_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS1_vec));
dS0_0123 = vget_low_s16 (qS0_vec);
dS1_0123 = vget_low_s16 (qS1_vec);
qT0_vec = vmull_s16 (dS0_0123, dCoeff);
qT1_vec = vmull_s16 (dS1_0123, dCoeff);
vst1q_s32 (&D0[dx], qT0_vec);
vst1q_s32 (&D1[dx], qT1_vec);
}
}
for (; k < count; k++)
{
const unsigned char *S = src[k];
int *D = dst[k];
for (dx = 0; dx < xmax; dx += 4)
{
int sx = xofs[dx];
alpha_vec = vld2_s16 (&alpha[dx * 2]);
dS0_vec = vld1_u8 (&S[sx]);
qS0_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS0_vec));
dS0_0123 = vget_low_s16 (qS0_vec);
dS0_4567 = vget_high_s16 (qS0_vec);
qT0_vec = vmull_s16 (dS0_0123, alpha_vec.val[0]);
qT0_vec = vmlal_s16 (qT0_vec, dS0_4567, alpha_vec.val[1]);
vst1q_s32 (&D[dx], qT0_vec);
}
for (; dx < dwidth; dx += 4)
{
int sx = xofs[dx];
dS0_vec = vld1_u8 (&S[sx]);
qS0_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS0_vec));
dS0_0123 = vget_low_s16 (qS0_vec);
qT0_vec = vmull_s16 (dS0_0123, dCoeff);
vst1q_s32 (&D[dx], qT0_vec);
}
}
}
void aom_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_x, // unused
int x_step_q4, // unused
const int16_t *filter_y, int y_step_q4, int w,
int h) {
int height;
const uint8_t *s;
uint8_t *d;
uint32x2_t d2u32, d3u32;
uint32x2_t d16u32, d18u32, d20u32, d22u32, d24u32, d26u32;
int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16;
int16x4_t d24s16, d25s16, d26s16, d27s16;
uint16x4_t d2u16, d3u16, d4u16, d5u16;
int16x8_t q0s16;
uint16x8_t q1u16, q2u16, q8u16, q9u16, q10u16, q11u16, q12u16, q13u16;
int32x4_t q1s32, q2s32, q14s32, q15s32;
assert(y_step_q4 == 16);
(void)x_step_q4;
(void)y_step_q4;
(void)filter_x;
src -= src_stride * 3;
q0s16 = vld1q_s16(filter_y);
for (; w > 0; w -= 4, src += 4, dst += 4) { // loop_vert_h
s = src;
d16u32 = vld1_lane_u32((const uint32_t *)s, d16u32, 0);
s += src_stride;
d16u32 = vld1_lane_u32((const uint32_t *)s, d16u32, 1);
s += src_stride;
d18u32 = vld1_lane_u32((const uint32_t *)s, d18u32, 0);
s += src_stride;
d18u32 = vld1_lane_u32((const uint32_t *)s, d18u32, 1);
s += src_stride;
d20u32 = vld1_lane_u32((const uint32_t *)s, d20u32, 0);
s += src_stride;
d20u32 = vld1_lane_u32((const uint32_t *)s, d20u32, 1);
s += src_stride;
d22u32 = vld1_lane_u32((const uint32_t *)s, d22u32, 0);
s += src_stride;
q8u16 = vmovl_u8(vreinterpret_u8_u32(d16u32));
q9u16 = vmovl_u8(vreinterpret_u8_u32(d18u32));
q10u16 = vmovl_u8(vreinterpret_u8_u32(d20u32));
q11u16 = vmovl_u8(vreinterpret_u8_u32(d22u32));
d18s16 = vreinterpret_s16_u16(vget_low_u16(q9u16));
d19s16 = vreinterpret_s16_u16(vget_high_u16(q9u16));
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d = dst;
for (height = h; height > 0; height -= 4) { // loop_vert
d24u32 = vld1_lane_u32((const uint32_t *)s, d24u32, 0);
s += src_stride;
d26u32 = vld1_lane_u32((const uint32_t *)s, d26u32, 0);
s += src_stride;
d26u32 = vld1_lane_u32((const uint32_t *)s, d26u32, 1);
s += src_stride;
d24u32 = vld1_lane_u32((const uint32_t *)s, d24u32, 1);
s += src_stride;
q12u16 = vmovl_u8(vreinterpret_u8_u32(d24u32));
q13u16 = vmovl_u8(vreinterpret_u8_u32(d26u32));
d16s16 = vreinterpret_s16_u16(vget_low_u16(q8u16));
d17s16 = vreinterpret_s16_u16(vget_high_u16(q8u16));
d20s16 = vreinterpret_s16_u16(vget_low_u16(q10u16));
d21s16 = vreinterpret_s16_u16(vget_high_u16(q10u16));
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
__builtin_prefetch(d);
__builtin_prefetch(d + dst_stride);
q1s32 = MULTIPLY_BY_Q0(d16s16, d17s16, d18s16, d19s16, d20s16, d21s16,
d22s16, d24s16, q0s16);
__builtin_prefetch(d + dst_stride * 2);
__builtin_prefetch(d + dst_stride * 3);
q2s32 = MULTIPLY_BY_Q0(d17s16, d18s16, d19s16, d20s16, d21s16, d22s16,
d24s16, d26s16, q0s16);
__builtin_prefetch(s);
__builtin_prefetch(s + src_stride);
q14s32 = MULTIPLY_BY_Q0(d18s16, d19s16, d20s16, d21s16, d22s16, d24s16,
d26s16, d27s16, q0s16);
__builtin_prefetch(s + src_stride * 2);
__builtin_prefetch(s + src_stride * 3);
q15s32 = MULTIPLY_BY_Q0(d19s16, d20s16, d21s16, d22s16, d24s16, d26s16,
d27s16, d25s16, q0s16);
d2u16 = vqrshrun_n_s32(q1s32, 7);
d3u16 = vqrshrun_n_s32(q2s32, 7);
d4u16 = vqrshrun_n_s32(q14s32, 7);
d5u16 = vqrshrun_n_s32(q15s32, 7);
q1u16 = vcombine_u16(d2u16, d3u16);
q2u16 = vcombine_u16(d4u16, d5u16);
d2u32 = vreinterpret_u32_u8(vqmovn_u16(q1u16));
d3u32 = vreinterpret_u32_u8(vqmovn_u16(q2u16));
vst1_lane_u32((uint32_t *)d, d2u32, 0);
d += dst_stride;
vst1_lane_u32((uint32_t *)d, d2u32, 1);
d += dst_stride;
vst1_lane_u32((uint32_t *)d, d3u32, 0);
d += dst_stride;
vst1_lane_u32((uint32_t *)d, d3u32, 1);
d += dst_stride;
q8u16 = q10u16;
d18s16 = d22s16;
d19s16 = d24s16;
q10u16 = q13u16;
d22s16 = d25s16;
}
}
return;
}
inline v_uint32x4 v_load_expand_q(const uchar* ptr)
{
uint8x8_t v0 = vcreate_u8(*(unsigned*)ptr);
uint16x4_t v1 = vget_low_u16(vmovl_u8(v0));
return v_uint32x4(vmovl_u16(v1));
}
XnStatus XnPacked12DepthProcessor::Unpack12to16(const XnUInt8* pcInput, const XnUInt32 nInputSize, XnUInt32* pnActualRead)
{
const XnUInt8* pOrigInput = pcInput;
XnUInt32 nElements = nInputSize / XN_INPUT_ELEMENT_SIZE; // floored
XnUInt32 nNeededOutput = nElements * XN_OUTPUT_ELEMENT_SIZE;
*pnActualRead = 0;
XnBuffer* pWriteBuffer = GetWriteBuffer();
if (!CheckDepthBufferForOverflow(nNeededOutput))
{
return XN_STATUS_OUTPUT_BUFFER_OVERFLOW;
}
XnUInt16* pnOutput = GetDepthOutputBuffer();
XnUInt16* pShiftOut = GetShiftsOutputBuffer();
XnUInt16 shift[16];
#ifdef XN_NEON
XnUInt16 depth[16];
uint8x8x3_t inD3;
uint8x8_t rshft4D, lshft4D;
uint16x8_t rshft4Q, lshft4Q;
uint16x8_t depthQ;
uint16x8x2_t shiftQ2;
#endif
// Convert the 11bit packed data into 16bit shorts
for (XnUInt32 nElem = 0; nElem < nElements; ++nElem)
{
#ifndef XN_NEON
// input: 0, 1,2,3, 4,5,6, 7,8,9, 10,11,12, 13,14,15, 16,17,18, 19,20,21, 22,23
// -,---,-,-,---,-,-,---,-,-,---,--,--,---,--,--,---,--,--,---,--,--,---,--
// bits: 8,4,4,8,8,4,4,8,8,4,4,8,8,4,4, 8, 8,4,4, 8, 8,4,4, 8, 8,4,4, 8, 8,4,4, 8
// ---,---,---,---,---,---,---,----,----,----,----,----,----,----,----,----
// output: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
shift[0] = (XN_TAKE_BITS(pcInput[0],8,0) << 4) | XN_TAKE_BITS(pcInput[1],4,4);
shift[1] = (XN_TAKE_BITS(pcInput[1],4,0) << 8) | XN_TAKE_BITS(pcInput[2],8,0);
shift[2] = (XN_TAKE_BITS(pcInput[3],8,0) << 4) | XN_TAKE_BITS(pcInput[4],4,4);
shift[3] = (XN_TAKE_BITS(pcInput[4],4,0) << 8) | XN_TAKE_BITS(pcInput[5],8,0);
shift[4] = (XN_TAKE_BITS(pcInput[6],8,0) << 4) | XN_TAKE_BITS(pcInput[7],4,4);
shift[5] = (XN_TAKE_BITS(pcInput[7],4,0) << 8) | XN_TAKE_BITS(pcInput[8],8,0);
shift[6] = (XN_TAKE_BITS(pcInput[9],8,0) << 4) | XN_TAKE_BITS(pcInput[10],4,4);
shift[7] = (XN_TAKE_BITS(pcInput[10],4,0) << 8) | XN_TAKE_BITS(pcInput[11],8,0);
shift[8] = (XN_TAKE_BITS(pcInput[12],8,0) << 4) | XN_TAKE_BITS(pcInput[13],4,4);
shift[9] = (XN_TAKE_BITS(pcInput[13],4,0) << 8) | XN_TAKE_BITS(pcInput[14],8,0);
shift[10] = (XN_TAKE_BITS(pcInput[15],8,0) << 4) | XN_TAKE_BITS(pcInput[16],4,4);
shift[11] = (XN_TAKE_BITS(pcInput[16],4,0) << 8) | XN_TAKE_BITS(pcInput[17],8,0);
shift[12] = (XN_TAKE_BITS(pcInput[18],8,0) << 4) | XN_TAKE_BITS(pcInput[19],4,4);
shift[13] = (XN_TAKE_BITS(pcInput[19],4,0) << 8) | XN_TAKE_BITS(pcInput[20],8,0);
shift[14] = (XN_TAKE_BITS(pcInput[21],8,0) << 4) | XN_TAKE_BITS(pcInput[22],4,4);
shift[15] = (XN_TAKE_BITS(pcInput[22],4,0) << 8) | XN_TAKE_BITS(pcInput[23],8,0);
pShiftOut[0] = (((shift[0]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[0]) : 0);
pShiftOut[1] = (((shift[1]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[1]) : 0);
pShiftOut[2] = (((shift[2]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[2]) : 0);
pShiftOut[3] = (((shift[3]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[3]) : 0);
pShiftOut[4] = (((shift[4]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[4]) : 0);
pShiftOut[5] = (((shift[5]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[5]) : 0);
pShiftOut[6] = (((shift[6]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[6]) : 0);
pShiftOut[7] = (((shift[7]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[7]) : 0);
pShiftOut[8] = (((shift[0]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[8]) : 0);
pShiftOut[9] = (((shift[1]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[9]) : 0);
pShiftOut[10] = (((shift[2]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[10]) : 0);
pShiftOut[11] = (((shift[3]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[11]) : 0);
pShiftOut[12] = (((shift[4]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[12]) : 0);
pShiftOut[13] = (((shift[5]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[13]) : 0);
pShiftOut[14] = (((shift[6]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[14]) : 0);
pShiftOut[15] = (((shift[7]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[15]) : 0);
pnOutput[0] = GetOutput(shift[0]);
pnOutput[1] = GetOutput(shift[1]);
pnOutput[2] = GetOutput(shift[2]);
pnOutput[3] = GetOutput(shift[3]);
pnOutput[4] = GetOutput(shift[4]);
pnOutput[5] = GetOutput(shift[5]);
pnOutput[6] = GetOutput(shift[6]);
pnOutput[7] = GetOutput(shift[7]);
pnOutput[8] = GetOutput(shift[8]);
pnOutput[9] = GetOutput(shift[9]);
pnOutput[10] = GetOutput(shift[10]);
pnOutput[11] = GetOutput(shift[11]);
pnOutput[12] = GetOutput(shift[12]);
pnOutput[13] = GetOutput(shift[13]);
pnOutput[14] = GetOutput(shift[14]);
pnOutput[15] = GetOutput(shift[15]);
#else
// input: 0, 1,2 (X8)
// -,---,-
// bits: 8,4,4,8 (X8)
// ---,---
// output: 0, 1 (X8)
// Split 24 bytes into 3 vectors (64 bit each)
inD3 = vld3_u8(pcInput);
// rshft4D0 contains 4 MSB of second vector (placed at offset 0)
rshft4D = vshr_n_u8(inD3.val[1], 4);
// lshft4D0 contains 4 LSB of second vector (placed at offset 4)
lshft4D = vshl_n_u8(inD3.val[1], 4);
// Expand 64 bit vectors to 128 bit (8 values of 16 bits)
shiftQ2.val[0] = vmovl_u8(inD3.val[0]);
shiftQ2.val[1] = vmovl_u8(inD3.val[2]);
rshft4Q = vmovl_u8(rshft4D);
lshft4Q = vmovl_u8(lshft4D);
// Even indexed shift = 8 bits from first vector + 4 MSB bits of second vector
shiftQ2.val[0] = vshlq_n_u16(shiftQ2.val[0], 4);
shiftQ2.val[0] = vorrq_u16(shiftQ2.val[0], rshft4Q);
// Odd indexed shift = 4 LSB bits of second vector + 8 bits from third vector
lshft4Q = vshlq_n_u16(lshft4Q, 4);
shiftQ2.val[1] = vorrq_u16(shiftQ2.val[1], lshft4Q);
// Interleave shift values to a single vector
vst2q_u16(shift, shiftQ2);
shift[0] = (((shift[0]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[0]) : 0);
shift[1] = (((shift[1]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[1]) : 0);
shift[2] = (((shift[2]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[2]) : 0);
shift[3] = (((shift[3]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[3]) : 0);
shift[4] = (((shift[4]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[4]) : 0);
shift[5] = (((shift[5]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[5]) : 0);
shift[6] = (((shift[6]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[6]) : 0);
shift[7] = (((shift[7]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[7]) : 0);
shift[8] = (((shift[0]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[8]) : 0);
shift[9] = (((shift[1]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[9]) : 0);
shift[10] = (((shift[2]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[10]) : 0);
shift[11] = (((shift[3]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[11]) : 0);
shift[12] = (((shift[4]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[12]) : 0);
shift[13] = (((shift[5]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[13]) : 0);
shift[14] = (((shift[6]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[14]) : 0);
shift[15] = (((shift[7]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[15]) : 0);
depth[0] = GetOutput(shift[0]);
depth[1] = GetOutput(shift[1]);
depth[2] = GetOutput(shift[2]);
depth[3] = GetOutput(shift[3]);
depth[4] = GetOutput(shift[4]);
depth[5] = GetOutput(shift[5]);
depth[6] = GetOutput(shift[6]);
depth[7] = GetOutput(shift[7]);
// Load
depthQ = vld1q_u16(depth);
//Store
vst1q_u16(pnOutput, depthQ);
// Load
depthQ = vld1q_u16(shift);
// Store
vst1q_u16(pShiftOut, depthQ);
depth[8] = GetOutput(shift[8]);
depth[9] = GetOutput(shift[9]);
depth[10] = GetOutput(shift[10]);
depth[11] = GetOutput(shift[11]);
depth[12] = GetOutput(shift[12]);
depth[13] = GetOutput(shift[13]);
depth[14] = GetOutput(shift[14]);
depth[15] = GetOutput(shift[15]);
// Load
depthQ = vld1q_u16(depth + 8);
// Store
vst1q_u16(pnOutput + 8, depthQ);
// Load
depthQ = vld1q_u16(shift + 8);
// Store
vst1q_u16(pShiftOut + 8, depthQ);
#endif
pcInput += XN_INPUT_ELEMENT_SIZE;
pnOutput += 16;
pShiftOut += 16;
}
*pnActualRead = (XnUInt32)(pcInput - pOrigInput);
pWriteBuffer->UnsafeUpdateSize(nNeededOutput);
return XN_STATUS_OK;
}
void aom_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, // unused
int y_step_q4, // unused
int w, int h) {
int width;
const uint8_t *s, *psrc;
uint8_t *d, *pdst;
uint8x8_t d2u8, d3u8, d24u8, d25u8, d26u8, d27u8, d28u8, d29u8;
uint32x2_t d2u32, d3u32, d28u32, d29u32, d30u32, d31u32;
uint8x16_t q12u8, q13u8, q14u8, q15u8;
int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d22s16, d23s16;
int16x4_t d24s16, d25s16, d26s16, d27s16;
uint16x4_t d2u16, d3u16, d4u16, d5u16, d16u16, d17u16, d18u16, d19u16;
int16x8_t q0s16;
uint16x8_t q1u16, q2u16, q8u16, q9u16, q10u16, q11u16, q12u16, q13u16;
int32x4_t q1s32, q2s32, q14s32, q15s32;
uint16x8x2_t q0x2u16;
uint8x8x2_t d0x2u8, d1x2u8;
uint32x2x2_t d0x2u32;
uint16x4x2_t d0x2u16, d1x2u16;
uint32x4x2_t q0x2u32;
assert(x_step_q4 == 16);
(void)x_step_q4;
(void)y_step_q4;
(void)filter_y;
q0s16 = vld1q_s16(filter_x);
src -= 3; // adjust for taps
for (; h > 0; h -= 4, src += src_stride * 4,
dst += dst_stride * 4) { // loop_horiz_v
s = src;
d24u8 = vld1_u8(s);
s += src_stride;
d25u8 = vld1_u8(s);
s += src_stride;
d26u8 = vld1_u8(s);
s += src_stride;
d27u8 = vld1_u8(s);
q12u8 = vcombine_u8(d24u8, d25u8);
q13u8 = vcombine_u8(d26u8, d27u8);
q0x2u16 =
vtrnq_u16(vreinterpretq_u16_u8(q12u8), vreinterpretq_u16_u8(q13u8));
d24u8 = vreinterpret_u8_u16(vget_low_u16(q0x2u16.val[0]));
d25u8 = vreinterpret_u8_u16(vget_high_u16(q0x2u16.val[0]));
d26u8 = vreinterpret_u8_u16(vget_low_u16(q0x2u16.val[1]));
d27u8 = vreinterpret_u8_u16(vget_high_u16(q0x2u16.val[1]));
d0x2u8 = vtrn_u8(d24u8, d25u8);
d1x2u8 = vtrn_u8(d26u8, d27u8);
__builtin_prefetch(src + src_stride * 4);
__builtin_prefetch(src + src_stride * 5);
__builtin_prefetch(src + src_stride * 6);
q8u16 = vmovl_u8(d0x2u8.val[0]);
q9u16 = vmovl_u8(d0x2u8.val[1]);
q10u16 = vmovl_u8(d1x2u8.val[0]);
q11u16 = vmovl_u8(d1x2u8.val[1]);
d16u16 = vget_low_u16(q8u16);
d17u16 = vget_high_u16(q8u16);
d18u16 = vget_low_u16(q9u16);
d19u16 = vget_high_u16(q9u16);
q8u16 = vcombine_u16(d16u16, d18u16); // vswp 17 18
q9u16 = vcombine_u16(d17u16, d19u16);
d20s16 = vreinterpret_s16_u16(vget_low_u16(q10u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q10u16)); // vmov 23 21
for (width = w, psrc = src + 7, pdst = dst; width > 0;
width -= 4, psrc += 4, pdst += 4) { // loop_horiz
s = psrc;
d28u32 = vld1_dup_u32((const uint32_t *)s);
s += src_stride;
d29u32 = vld1_dup_u32((const uint32_t *)s);
s += src_stride;
d31u32 = vld1_dup_u32((const uint32_t *)s);
s += src_stride;
d30u32 = vld1_dup_u32((const uint32_t *)s);
__builtin_prefetch(psrc + 64);
d0x2u16 =
vtrn_u16(vreinterpret_u16_u32(d28u32), vreinterpret_u16_u32(d31u32));
d1x2u16 =
vtrn_u16(vreinterpret_u16_u32(d29u32), vreinterpret_u16_u32(d30u32));
d0x2u8 = vtrn_u8(vreinterpret_u8_u16(d0x2u16.val[0]), // d28
vreinterpret_u8_u16(d1x2u16.val[0])); // d29
d1x2u8 = vtrn_u8(vreinterpret_u8_u16(d0x2u16.val[1]), // d31
vreinterpret_u8_u16(d1x2u16.val[1])); // d30
__builtin_prefetch(psrc + 64 + src_stride);
q14u8 = vcombine_u8(d0x2u8.val[0], d0x2u8.val[1]);
q15u8 = vcombine_u8(d1x2u8.val[1], d1x2u8.val[0]);
q0x2u32 =
vtrnq_u32(vreinterpretq_u32_u8(q14u8), vreinterpretq_u32_u8(q15u8));
d28u8 = vreinterpret_u8_u32(vget_low_u32(q0x2u32.val[0]));
d29u8 = vreinterpret_u8_u32(vget_high_u32(q0x2u32.val[0]));
q12u16 = vmovl_u8(d28u8);
q13u16 = vmovl_u8(d29u8);
__builtin_prefetch(psrc + 64 + src_stride * 2);
d16s16 = vreinterpret_s16_u16(vget_low_u16(q8u16));
d17s16 = vreinterpret_s16_u16(vget_high_u16(q8u16));
d18s16 = vreinterpret_s16_u16(vget_low_u16(q9u16));
d19s16 = vreinterpret_s16_u16(vget_high_u16(q9u16));
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
q1s32 = MULTIPLY_BY_Q0(d16s16, d17s16, d20s16, d22s16, d18s16, d19s16,
d23s16, d24s16, q0s16);
q2s32 = MULTIPLY_BY_Q0(d17s16, d20s16, d22s16, d18s16, d19s16, d23s16,
d24s16, d26s16, q0s16);
q14s32 = MULTIPLY_BY_Q0(d20s16, d22s16, d18s16, d19s16, d23s16, d24s16,
d26s16, d27s16, q0s16);
q15s32 = MULTIPLY_BY_Q0(d22s16, d18s16, d19s16, d23s16, d24s16, d26s16,
d27s16, d25s16, q0s16);
__builtin_prefetch(psrc + 60 + src_stride * 3);
d2u16 = vqrshrun_n_s32(q1s32, 7);
d3u16 = vqrshrun_n_s32(q2s32, 7);
d4u16 = vqrshrun_n_s32(q14s32, 7);
d5u16 = vqrshrun_n_s32(q15s32, 7);
q1u16 = vcombine_u16(d2u16, d3u16);
q2u16 = vcombine_u16(d4u16, d5u16);
d2u8 = vqmovn_u16(q1u16);
d3u8 = vqmovn_u16(q2u16);
d0x2u16 = vtrn_u16(vreinterpret_u16_u8(d2u8), vreinterpret_u16_u8(d3u8));
d0x2u32 = vtrn_u32(vreinterpret_u32_u16(d0x2u16.val[0]),
vreinterpret_u32_u16(d0x2u16.val[1]));
d0x2u8 = vtrn_u8(vreinterpret_u8_u32(d0x2u32.val[0]),
vreinterpret_u8_u32(d0x2u32.val[1]));
d2u32 = vreinterpret_u32_u8(d0x2u8.val[0]);
d3u32 = vreinterpret_u32_u8(d0x2u8.val[1]);
d = pdst;
vst1_lane_u32((uint32_t *)d, d2u32, 0);
d += dst_stride;
vst1_lane_u32((uint32_t *)d, d3u32, 0);
d += dst_stride;
vst1_lane_u32((uint32_t *)d, d2u32, 1);
d += dst_stride;
vst1_lane_u32((uint32_t *)d, d3u32, 1);
q8u16 = q9u16;
d20s16 = d23s16;
q11u16 = q12u16;
q9u16 = q13u16;
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
}
}
return;
}
inline uint16x8_t vmovl(const uint8x8_t & v) { return vmovl_u8(v); }
SIMD_INLINE uint8x8_t BgraToGray(uint8x8x4_t bgra)
{
return vmovn_u16(BgrToGray(vmovl_u8(bgra.val[0]), vmovl_u8(bgra.val[1]), vmovl_u8(bgra.val[2])));
}
