void test_vmovQ_nu32 (void)
{
uint32x4_t out_uint32x4_t;
uint32_t arg0_uint32_t;
out_uint32x4_t = vmovq_n_u32 (arg0_uint32_t);
}
/* u32x4 mm mul */
void mw_neon_mm_mul_u32x4(unsigned int * A, int Row, int T, unsigned int * B, int Col, unsigned int * C)
{
int i, k, j;
uint32x4_t neon_b, neon_c;
uint32x4_t neon_a0, neon_a1, neon_a2, neon_a3;
uint32x4_t neon_b0, neon_b1, neon_b2, neon_b3;
for (i = 0; i < Row; i+=4)
{
for (k = 0; k < Col; k+=1)
{
neon_c = vmovq_n_u32(0);
for (j = 0; j < T; j+=4)
{
int j_T = j * T + i;
int k_Row = k * Row;
neon_a0 = vld1q_u32(A + j_T);
j_T+=Row;
neon_a1 = vld1q_u32(A + j_T);
j_T+=Row;
neon_a2 = vld1q_u32(A + j_T);
j_T+=Row;
neon_a3 = vld1q_u32(A + j_T);
neon_b = vld1q_u32(B + k_Row + j);
neon_b0 = vdupq_n_u32(vgetq_lane_u32(neon_b, 0));
neon_b1 = vdupq_n_u32(vgetq_lane_u32(neon_b, 1));
neon_b2 = vdupq_n_u32(vgetq_lane_u32(neon_b, 2));
neon_b3 = vdupq_n_u32(vgetq_lane_u32(neon_b, 3));
neon_c = vaddq_u32(vmulq_u32(neon_a0, neon_b0), neon_c);
neon_c = vaddq_u32(vmulq_u32(neon_a1, neon_b1), neon_c);
neon_c = vaddq_u32(vmulq_u32(neon_a2, neon_b2), neon_c);
neon_c = vaddq_u32(vmulq_u32(neon_a3, neon_b3), neon_c);
vst1q_lane_u32(C + k_Row + i, neon_c, 0);
vst1q_lane_u32(C + k_Row + i + 1, neon_c, 1);
vst1q_lane_u32(C + k_Row + i + 2, neon_c, 2);
vst1q_lane_u32(C + k_Row + i + 3, neon_c, 3);
}
}
}
}
/* u32x4 mv mul */
void mw_neon_mv_mul_u32x4(unsigned int * A, int Row, int T, unsigned int * B, unsigned int * C)
{
int i = 0;
int k = 0;
uint32x4_t neon_b, neon_c;
uint32x4_t neon_a0, neon_a1, neon_a2, neon_a3;
uint32x4_t neon_b0, neon_b1, neon_b2, neon_b3;
for (i = 0; i < Row; i+=4)
{
neon_c = vmovq_n_u32(0);
for (k = 0; k < T; k+=4)
{
int j = k * T + i;
neon_a0 = vld1q_u32(A + j);
j+=Row;
neon_a1 = vld1q_u32(A + j);
j+=Row;
neon_a2 = vld1q_u32(A + j);
j+=Row;
neon_a3 = vld1q_u32(A + j);
neon_b = vld1q_u32(B + k);
neon_b0 = vdupq_n_u32(vgetq_lane_u32(neon_b, 0));
neon_b1 = vdupq_n_u32(vgetq_lane_u32(neon_b, 1));
neon_b2 = vdupq_n_u32(vgetq_lane_u32(neon_b, 2));
neon_b3 = vdupq_n_u32(vgetq_lane_u32(neon_b, 3));
neon_c = vaddq_u32(vmulq_u32(neon_a0, neon_b0), neon_c);
neon_c = vaddq_u32(vmulq_u32(neon_a1, neon_b1), neon_c);
neon_c = vaddq_u32(vmulq_u32(neon_a2, neon_b2), neon_c);
neon_c = vaddq_u32(vmulq_u32(neon_a3, neon_b3), neon_c);
}
vst1q_u32(C + i, neon_c);
}
}
uint32x4_t test_vmovq_n_u32(uint32_t v1) {
// CHECK: test_vmovq_n_u32
return vmovq_n_u32(v1);
// CHECK: dup {{v[0-9]+}}.4s, {{w[0-9]+}}
}
inline ResultType operator()(Iterator1 a, Iterator2 b, size_t size) const
{
ResultType result = 0;
#if (defined __GNUC__ || defined __clang__) && defined USE_SSE
#ifdef __ARM_NEON__
{
uint32x4_t bits = vmovq_n_u32(0);
for (size_t i = 0; i < size; i += 16) {
uint8x16_t A_vec = vld1q_u8 (a + i);
uint8x16_t B_vec = vld1q_u8 (b + i);
uint8x16_t AxorB = veorq_u8 (A_vec, B_vec);
uint8x16_t bitsSet = vcntq_u8 (AxorB);
uint16x8_t bitSet8 = vpaddlq_u8 (bitsSet);
uint32x4_t bitSet4 = vpaddlq_u16 (bitSet8);
bits = vaddq_u32(bits, bitSet4);
}
uint64x2_t bitSet2 = vpaddlq_u32 (bits);
result = vgetq_lane_s32 (vreinterpretq_s32_u64(bitSet2),0);
result += vgetq_lane_s32 (vreinterpretq_s32_u64(bitSet2),2);
}
#else
{
//for portability just use unsigned long -- and use the __builtin_popcountll (see docs for __builtin_popcountll)
typedef unsigned long long pop_t;
const size_t modulo = size % sizeof(pop_t);
const pop_t* a2 = reinterpret_cast<const pop_t*> (a);
const pop_t* b2 = reinterpret_cast<const pop_t*> (b);
const pop_t* a2_end = a2 + (size / sizeof(pop_t));
for (; a2 != a2_end; ++a2, ++b2) result += __builtin_popcountll((*a2) ^ (*b2));
if (modulo) {
//in the case where size is not dividable by sizeof(pop_t)
//need to mask off the bits at the end
pop_t a_final = 0, b_final = 0;
memcpy(&a_final, a2, modulo);
memcpy(&b_final, b2, modulo);
result += __builtin_popcountll(a_final ^ b_final);
}
}
#endif //NEON
return result;
#endif
#ifdef PLATFORM_64_BIT
if(size%64 == 0)
{
const uint64_t* pa = reinterpret_cast<const uint64_t*>(a);
const uint64_t* pb = reinterpret_cast<const uint64_t*>(b);
size /= (sizeof(uint64_t)/sizeof(unsigned char));
for(size_t i = 0; i < size; ++i, ++pa, ++pb ) {
result += popcnt64(*pa ^ *pb);
}
}
else
{
const uint32_t* pa = reinterpret_cast<const uint32_t*>(a);
const uint32_t* pb = reinterpret_cast<const uint32_t*>(b);
size /= (sizeof(uint32_t)/sizeof(unsigned char));
for(size_t i = 0; i < size; ++i, ++pa, ++pb ) {
result += popcnt32(*pa ^ *pb);
}
}
#else
const uint32_t* pa = reinterpret_cast<const uint32_t*>(a);
const uint32_t* pb = reinterpret_cast<const uint32_t*>(b);
size /= (sizeof(uint32_t)/sizeof(unsigned char));
for(size_t i = 0; i < size; ++i, ++pa, ++pb ) {
result += popcnt32(*pa ^ *pb);
}
#endif
return result;
}
f64 dotProduct(const Size2D &_size,
const u8 * src0Base, ptrdiff_t src0Stride,
const u8 * src1Base, ptrdiff_t src1Stride)
{
internal::assertSupportedConfiguration();
#ifdef CAROTENE_NEON
Size2D size(_size);
if (src0Stride == src1Stride &&
src0Stride == (ptrdiff_t)(size.width))
{
size.width *= size.height;
size.height = 1;
}
// It is possible to accumulate up to 66051 uchar multiplication results in uint32 without overflow
// We process 16 elements and accumulate two new elements per step. So we could handle 66051/2*16 elements
#define DOT_UINT_BLOCKSIZE 66050*8
f64 result = 0.0;
for (size_t row = 0; row < size.height; ++row)
{
const u8 * src0 = internal::getRowPtr(src0Base, src0Stride, row);
const u8 * src1 = internal::getRowPtr(src1Base, src1Stride, row);
size_t i = 0;
uint64x2_t ws = vmovq_n_u64(0);
while(i + 16 <= size.width)
{
size_t lim = std::min(i + DOT_UINT_BLOCKSIZE, size.width) - 16;
uint32x4_t s1 = vmovq_n_u32(0);
uint32x4_t s2 = vmovq_n_u32(0);
for (; i <= lim; i += 16)
{
internal::prefetch(src0 + i);
internal::prefetch(src1 + i);
uint8x16_t vs1 = vld1q_u8(src0 + i);
uint8x16_t vs2 = vld1q_u8(src1 + i);
uint16x8_t vdot1 = vmull_u8(vget_low_u8(vs1), vget_low_u8(vs2));
uint16x8_t vdot2 = vmull_u8(vget_high_u8(vs1), vget_high_u8(vs2));
s1 = vpadalq_u16(s1, vdot1);
s2 = vpadalq_u16(s2, vdot2);
}
ws = vpadalq_u32(ws, s1);
ws = vpadalq_u32(ws, s2);
}
if(i + 8 <= size.width)
{
uint8x8_t vs1 = vld1_u8(src0 + i);
uint8x8_t vs2 = vld1_u8(src1 + i);
ws = vpadalq_u32(ws, vpaddlq_u16(vmull_u8(vs1, vs2)));
i += 8;
}
result += (double)vget_lane_u64(vadd_u64(vget_low_u64(ws), vget_high_u64(ws)), 0);
for (; i < size.width; ++i)
result += s32(src0[i]) * s32(src1[i]);
}
return result;
#else
(void)_size;
(void)src0Base;
(void)src0Stride;
(void)src1Base;
(void)src1Stride;
return 0;
#endif
}
