Ejemplo n.º 1
0
void test_vaddQu8 (void)
{
  uint8x16_t out_uint8x16_t;
  uint8x16_t arg0_uint8x16_t;
  uint8x16_t arg1_uint8x16_t;

  out_uint8x16_t = vaddq_u8 (arg0_uint8x16_t, arg1_uint8x16_t);
}
Ejemplo n.º 2
0
void add3 (uint8x16_t *data) {
    /* Set each sixteen values of the vector to 3.
     *
     * Remark: a 'q' suffix to intrinsics indicates
     * the instruction run for 128 bits registers.
     */
    uint8x16_t three = vmovq_n_u8 (3);

    /* Add 3 to the value given in argument. */
    *data = vaddq_u8 (*data, three);
}
Ejemplo n.º 3
0
static void AddGreenToBlueAndRed(uint32_t* argb_data, int num_pixels) {
  const uint32_t* const end = argb_data + (num_pixels & ~3);
  const uint8x8_t shuffle = vld1_u8(kGreenShuffle);
  for (; argb_data < end; argb_data += 4) {
    const uint8x16_t argb = vld1q_u8((uint8_t*)argb_data);
    const uint8x16_t greens =
        vcombine_u8(vtbl1_u8(vget_low_u8(argb), shuffle),
                    vtbl1_u8(vget_high_u8(argb), shuffle));
    vst1q_u8((uint8_t*)argb_data, vaddq_u8(argb, greens));
  }
  // fallthrough and finish off with plain-C
  VP8LAddGreenToBlueAndRed_C(argb_data, num_pixels & 3);
}
Ejemplo n.º 4
0
Archivo: ops.c Proyecto: petecoup/argon
void ar_vadd_u8_neon(uint8_t* res,
                     const uint8_t* a,
                     const uint8_t* b,
                     uint32_t n)
{
   uint8x16_t a_loaded;
   uint8x16_t b_loaded;
   uint8x16_t res_loaded;

   for (uint32_t i = 0; i < n; i += 16) {
      a_loaded = vld1q_u8(&(a[i]));
      b_loaded = vld1q_u8(&(b[i]));
      res_loaded = vaddq_u8(a_loaded, b_loaded);
      vst1q_u8(&(res[i]),res_loaded);
   }
}
Ejemplo n.º 5
0
void
png_read_filter_row_up_neon(png_row_infop row_info, png_bytep row,
   png_const_bytep prev_row)
{
   png_bytep rp = row;
   png_bytep rp_stop = row + row_info->rowbytes;
   png_const_bytep pp = prev_row;

   for (; rp < rp_stop; rp += 16, pp += 16)
   {
      uint8x16_t qrp, qpp;

      qrp = vld1q_u8(rp);
      qpp = vld1q_u8(pp);
      qrp = vaddq_u8(qrp, qpp);
      vst1q_u8(rp, qrp);
   }
}
Ejemplo n.º 6
0
uint64_t popcnt_neon_vcnt(const uint8_t* data, const size_t size)
{
    const size_t chunk_size = 16 * 4 * 2;

    uint8_t* ptr = const_cast<uint8_t*>(data);

    const size_t n = size / chunk_size;
    const size_t k = size % chunk_size;

    uint32x4_t sum = vcombine_u32(vcreate_u32(0), vcreate_u32(0));

    for (size_t i=0; i < n; i++, ptr += chunk_size) {

        uint8x16x4_t input0 = vld4q_u8(ptr + 0 * 16 * 4);
        uint8x16x4_t input1 = vld4q_u8(ptr + 1 * 16 * 4);

        uint8x16_t t0   = vcntq_u8(input0.val[0]);
        t0 = vaddq_u8(t0, vcntq_u8(input0.val[1]));
        t0 = vaddq_u8(t0, vcntq_u8(input0.val[2]));
        t0 = vaddq_u8(t0, vcntq_u8(input0.val[3]));

        t0 = vaddq_u8(t0, vcntq_u8(input1.val[0]));
        t0 = vaddq_u8(t0, vcntq_u8(input1.val[1]));
        t0 = vaddq_u8(t0, vcntq_u8(input1.val[2]));
        t0 = vaddq_u8(t0, vcntq_u8(input1.val[3]));

        const uint16x8_t t1 = vpaddlq_u8(t0);

        sum = vpadalq_u16(sum, t1);
    }

    uint32_t scalar = 0;
    uint32_t tmp[4];

    vst1q_u32(tmp, sum);
    for (int i=0; i < 4; i++) {
        scalar += tmp[i];
    }

    for (size_t j=0; j < k; j++) {
        scalar += lookup8bit[ptr[j]];
    }

    return scalar;
}
Ejemplo n.º 7
0
/* u8x16 add */
void mw_neon_mm_add_u8x16(unsigned char * A, int Row, int Col, unsigned char * B, unsigned char * C)
{
	uint8x16_t neon_a, neon_b, neon_c;
	int size = Row * Col;
	int i = 0;
	int k = 0;

	for (i = 16; i <= size ; i+=16)
	{
		k = i - 16;
		neon_a = vld1q_u8(A + k);
		neon_b = vld1q_u8(B + k);
		neon_c = vaddq_u8(neon_a, neon_b);
		vst1q_u8(C + k, neon_c);
	}

	k = i - 16;
    for (i = 0; i < size % 16; i++)
	{
		C[k + i] = A[k + i] + B[k + i];
	}
}
Ejemplo n.º 8
0
int vp8_denoiser_filter_neon(YV12_BUFFER_CONFIG *mc_running_avg,
                             YV12_BUFFER_CONFIG *running_avg,
                             MACROBLOCK *signal, unsigned int motion_magnitude,
                             int y_offset, int uv_offset) {
    /* If motion_magnitude is small, making the denoiser more aggressive by
     * increasing the adjustment for each level, level1 adjustment is
     * increased, the deltas stay the same.
     */
    const uint8x16_t v_level1_adjustment = vdupq_n_u8(
        (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) ? 4 : 3);
    const uint8x16_t v_delta_level_1_and_2 = vdupq_n_u8(1);
    const uint8x16_t v_delta_level_2_and_3 = vdupq_n_u8(2);
    const uint8x16_t v_level1_threshold = vdupq_n_u8(4);
    const uint8x16_t v_level2_threshold = vdupq_n_u8(8);
    const uint8x16_t v_level3_threshold = vdupq_n_u8(16);

    /* Local variables for array pointers and strides. */
    unsigned char *sig = signal->thismb;
    int            sig_stride = 16;
    unsigned char *mc_running_avg_y = mc_running_avg->y_buffer + y_offset;
    int            mc_running_avg_y_stride = mc_running_avg->y_stride;
    unsigned char *running_avg_y = running_avg->y_buffer + y_offset;
    int            running_avg_y_stride = running_avg->y_stride;

    /* Go over lines. */
    int i;
    int sum_diff = 0;
    for (i = 0; i < 16; ++i) {
        int8x16_t v_sum_diff = vdupq_n_s8(0);
        uint8x16_t v_running_avg_y;

        /* Load inputs. */
        const uint8x16_t v_sig = vld1q_u8(sig);
        const uint8x16_t v_mc_running_avg_y = vld1q_u8(mc_running_avg_y);

        /* Calculate absolute difference and sign masks. */
        const uint8x16_t v_abs_diff      = vabdq_u8(v_sig, v_mc_running_avg_y);
        const uint8x16_t v_diff_pos_mask = vcltq_u8(v_sig, v_mc_running_avg_y);
        const uint8x16_t v_diff_neg_mask = vcgtq_u8(v_sig, v_mc_running_avg_y);

        /* Figure out which level that put us in. */
        const uint8x16_t v_level1_mask = vcleq_u8(v_level1_threshold,
                                                  v_abs_diff);
        const uint8x16_t v_level2_mask = vcleq_u8(v_level2_threshold,
                                                  v_abs_diff);
        const uint8x16_t v_level3_mask = vcleq_u8(v_level3_threshold,
                                                  v_abs_diff);

        /* Calculate absolute adjustments for level 1, 2 and 3. */
        const uint8x16_t v_level2_adjustment = vandq_u8(v_level2_mask,
                                                        v_delta_level_1_and_2);
        const uint8x16_t v_level3_adjustment = vandq_u8(v_level3_mask,
                                                        v_delta_level_2_and_3);
        const uint8x16_t v_level1and2_adjustment = vaddq_u8(v_level1_adjustment,
            v_level2_adjustment);
        const uint8x16_t v_level1and2and3_adjustment = vaddq_u8(
            v_level1and2_adjustment, v_level3_adjustment);

        /* Figure adjustment absolute value by selecting between the absolute
         * difference if in level0 or the value for level 1, 2 and 3.
         */
        const uint8x16_t v_abs_adjustment = vbslq_u8(v_level1_mask,
            v_level1and2and3_adjustment, v_abs_diff);

        /* Calculate positive and negative adjustments. Apply them to the signal
         * and accumulate them. Adjustments are less than eight and the maximum
         * sum of them (7 * 16) can fit in a signed char.
         */
        const uint8x16_t v_pos_adjustment = vandq_u8(v_diff_pos_mask,
                                                     v_abs_adjustment);
        const uint8x16_t v_neg_adjustment = vandq_u8(v_diff_neg_mask,
                                                     v_abs_adjustment);
        v_running_avg_y = vqaddq_u8(v_sig, v_pos_adjustment);
        v_running_avg_y = vqsubq_u8(v_running_avg_y, v_neg_adjustment);
        v_sum_diff = vqaddq_s8(v_sum_diff,
                               vreinterpretq_s8_u8(v_pos_adjustment));
        v_sum_diff = vqsubq_s8(v_sum_diff,
                               vreinterpretq_s8_u8(v_neg_adjustment));

        /* Store results. */
        vst1q_u8(running_avg_y, v_running_avg_y);

        /* Sum all the accumulators to have the sum of all pixel differences
         * for this macroblock.
         */
        {
            int s0 = vgetq_lane_s8(v_sum_diff,  0) +
                     vgetq_lane_s8(v_sum_diff,  1) +
                     vgetq_lane_s8(v_sum_diff,  2) +
                     vgetq_lane_s8(v_sum_diff,  3);
            int s1 = vgetq_lane_s8(v_sum_diff,  4) +
                     vgetq_lane_s8(v_sum_diff,  5) +
                     vgetq_lane_s8(v_sum_diff,  6) +
                     vgetq_lane_s8(v_sum_diff,  7);
            int s2 = vgetq_lane_s8(v_sum_diff,  8) +
                     vgetq_lane_s8(v_sum_diff,  9) +
                     vgetq_lane_s8(v_sum_diff, 10) +
                     vgetq_lane_s8(v_sum_diff, 11);
            int s3 = vgetq_lane_s8(v_sum_diff, 12) +
                     vgetq_lane_s8(v_sum_diff, 13) +
                     vgetq_lane_s8(v_sum_diff, 14) +
                     vgetq_lane_s8(v_sum_diff, 15);
            sum_diff += s0 + s1+ s2 + s3;
        }

        /* Update pointers for next iteration. */
        sig += sig_stride;
        mc_running_avg_y += mc_running_avg_y_stride;
        running_avg_y += running_avg_y_stride;
    }

    /* Too much adjustments => copy block. */
    if (abs(sum_diff) > SUM_DIFF_THRESHOLD)
        return COPY_BLOCK;

    /* Tell above level that block was filtered. */
    vp8_copy_mem16x16(running_avg->y_buffer + y_offset, running_avg_y_stride,
                      signal->thismb, sig_stride);
    return FILTER_BLOCK;
}
Ejemplo n.º 9
0
int vp8_denoiser_filter_neon(unsigned char *mc_running_avg_y,
                             int mc_running_avg_y_stride,
                             unsigned char *running_avg_y,
                             int running_avg_y_stride,
                             unsigned char *sig, int sig_stride,
                             unsigned int motion_magnitude,
                             int increase_denoising) {
    /* If motion_magnitude is small, making the denoiser more aggressive by
     * increasing the adjustment for each level, level1 adjustment is
     * increased, the deltas stay the same.
     */
    int shift_inc  = (increase_denoising &&
        motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) ? 1 : 0;
    const uint8x16_t v_level1_adjustment = vmovq_n_u8(
        (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) ? 4 + shift_inc : 3);
    const uint8x16_t v_delta_level_1_and_2 = vdupq_n_u8(1);
    const uint8x16_t v_delta_level_2_and_3 = vdupq_n_u8(2);
    const uint8x16_t v_level1_threshold = vmovq_n_u8(4 + shift_inc);
    const uint8x16_t v_level2_threshold = vdupq_n_u8(8);
    const uint8x16_t v_level3_threshold = vdupq_n_u8(16);
    int64x2_t v_sum_diff_total = vdupq_n_s64(0);

    /* Go over lines. */
    int r;
    for (r = 0; r < 16; ++r) {
        /* Load inputs. */
        const uint8x16_t v_sig = vld1q_u8(sig);
        const uint8x16_t v_mc_running_avg_y = vld1q_u8(mc_running_avg_y);

        /* Calculate absolute difference and sign masks. */
        const uint8x16_t v_abs_diff      = vabdq_u8(v_sig, v_mc_running_avg_y);
        const uint8x16_t v_diff_pos_mask = vcltq_u8(v_sig, v_mc_running_avg_y);
        const uint8x16_t v_diff_neg_mask = vcgtq_u8(v_sig, v_mc_running_avg_y);

        /* Figure out which level that put us in. */
        const uint8x16_t v_level1_mask = vcleq_u8(v_level1_threshold,
                                                  v_abs_diff);
        const uint8x16_t v_level2_mask = vcleq_u8(v_level2_threshold,
                                                  v_abs_diff);
        const uint8x16_t v_level3_mask = vcleq_u8(v_level3_threshold,
                                                  v_abs_diff);

        /* Calculate absolute adjustments for level 1, 2 and 3. */
        const uint8x16_t v_level2_adjustment = vandq_u8(v_level2_mask,
                                                        v_delta_level_1_and_2);
        const uint8x16_t v_level3_adjustment = vandq_u8(v_level3_mask,
                                                        v_delta_level_2_and_3);
        const uint8x16_t v_level1and2_adjustment = vaddq_u8(v_level1_adjustment,
            v_level2_adjustment);
        const uint8x16_t v_level1and2and3_adjustment = vaddq_u8(
            v_level1and2_adjustment, v_level3_adjustment);

        /* Figure adjustment absolute value by selecting between the absolute
         * difference if in level0 or the value for level 1, 2 and 3.
         */
        const uint8x16_t v_abs_adjustment = vbslq_u8(v_level1_mask,
            v_level1and2and3_adjustment, v_abs_diff);

        /* Calculate positive and negative adjustments. Apply them to the signal
         * and accumulate them. Adjustments are less than eight and the maximum
         * sum of them (7 * 16) can fit in a signed char.
         */
        const uint8x16_t v_pos_adjustment = vandq_u8(v_diff_pos_mask,
                                                     v_abs_adjustment);
        const uint8x16_t v_neg_adjustment = vandq_u8(v_diff_neg_mask,
                                                     v_abs_adjustment);

        uint8x16_t v_running_avg_y = vqaddq_u8(v_sig, v_pos_adjustment);
        v_running_avg_y = vqsubq_u8(v_running_avg_y, v_neg_adjustment);

        /* Store results. */
        vst1q_u8(running_avg_y, v_running_avg_y);

        /* Sum all the accumulators to have the sum of all pixel differences
         * for this macroblock.
         */
        {
            const int8x16_t v_sum_diff =
                vqsubq_s8(vreinterpretq_s8_u8(v_pos_adjustment),
                          vreinterpretq_s8_u8(v_neg_adjustment));

            const int16x8_t fe_dc_ba_98_76_54_32_10 = vpaddlq_s8(v_sum_diff);

            const int32x4_t fedc_ba98_7654_3210 =
                vpaddlq_s16(fe_dc_ba_98_76_54_32_10);

            const int64x2_t fedcba98_76543210 =
                vpaddlq_s32(fedc_ba98_7654_3210);

            v_sum_diff_total = vqaddq_s64(v_sum_diff_total, fedcba98_76543210);
        }

        /* Update pointers for next iteration. */
        sig += sig_stride;
        mc_running_avg_y += mc_running_avg_y_stride;
        running_avg_y += running_avg_y_stride;
    }

    /* Too much adjustments => copy block. */
    {
        int64x1_t x = vqadd_s64(vget_high_s64(v_sum_diff_total),
                                      vget_low_s64(v_sum_diff_total));
        int sum_diff = vget_lane_s32(vabs_s32(vreinterpret_s32_s64(x)), 0);
        int sum_diff_thresh = SUM_DIFF_THRESHOLD;

        if (increase_denoising) sum_diff_thresh = SUM_DIFF_THRESHOLD_HIGH;
        if (sum_diff > sum_diff_thresh) {
          // Before returning to copy the block (i.e., apply no denoising),
          // checK if we can still apply some (weaker) temporal filtering to
          // this block, that would otherwise not be denoised at all. Simplest
          // is to apply an additional adjustment to running_avg_y to bring it
          // closer to sig. The adjustment is capped by a maximum delta, and
          // chosen such that in most cases the resulting sum_diff will be
          // within the accceptable range given by sum_diff_thresh.

          // The delta is set by the excess of absolute pixel diff over the
          // threshold.
          int delta = ((sum_diff - sum_diff_thresh) >> 8) + 1;
          // Only apply the adjustment for max delta up to 3.
          if (delta < 4) {
            const uint8x16_t k_delta = vmovq_n_u8(delta);
            sig -= sig_stride * 16;
            mc_running_avg_y -= mc_running_avg_y_stride * 16;
            running_avg_y -= running_avg_y_stride * 16;
            for (r = 0; r < 16; ++r) {
              uint8x16_t v_running_avg_y = vld1q_u8(running_avg_y);
              const uint8x16_t v_sig = vld1q_u8(sig);
              const uint8x16_t v_mc_running_avg_y = vld1q_u8(mc_running_avg_y);

              /* Calculate absolute difference and sign masks. */
              const uint8x16_t v_abs_diff      = vabdq_u8(v_sig,
                                                          v_mc_running_avg_y);
              const uint8x16_t v_diff_pos_mask = vcltq_u8(v_sig,
                                                          v_mc_running_avg_y);
              const uint8x16_t v_diff_neg_mask = vcgtq_u8(v_sig,
                                                          v_mc_running_avg_y);
              // Clamp absolute difference to delta to get the adjustment.
              const uint8x16_t v_abs_adjustment =
                  vminq_u8(v_abs_diff, (k_delta));

              const uint8x16_t v_pos_adjustment = vandq_u8(v_diff_pos_mask,
                                                           v_abs_adjustment);
              const uint8x16_t v_neg_adjustment = vandq_u8(v_diff_neg_mask,
                                                           v_abs_adjustment);

              v_running_avg_y = vqsubq_u8(v_running_avg_y, v_pos_adjustment);
              v_running_avg_y = vqaddq_u8(v_running_avg_y, v_neg_adjustment);

              /* Store results. */
              vst1q_u8(running_avg_y, v_running_avg_y);

              {
                  const int8x16_t v_sum_diff =
                      vqsubq_s8(vreinterpretq_s8_u8(v_neg_adjustment),
                                vreinterpretq_s8_u8(v_pos_adjustment));

                  const int16x8_t fe_dc_ba_98_76_54_32_10 =
                      vpaddlq_s8(v_sum_diff);
                  const int32x4_t fedc_ba98_7654_3210 =
                      vpaddlq_s16(fe_dc_ba_98_76_54_32_10);
                  const int64x2_t fedcba98_76543210 =
                      vpaddlq_s32(fedc_ba98_7654_3210);

                  v_sum_diff_total = vqaddq_s64(v_sum_diff_total,
                                                fedcba98_76543210);
              }
              /* Update pointers for next iteration. */
              sig += sig_stride;
              mc_running_avg_y += mc_running_avg_y_stride;
              running_avg_y += running_avg_y_stride;
            }
            {
              // Update the sum of all pixel differences of this MB.
              x = vqadd_s64(vget_high_s64(v_sum_diff_total),
                            vget_low_s64(v_sum_diff_total));
              sum_diff = vget_lane_s32(vabs_s32(vreinterpret_s32_s64(x)), 0);

              if (sum_diff > sum_diff_thresh) {
                return COPY_BLOCK;
              }
            }
          } else {
            return COPY_BLOCK;
          }
        }
    }
Ejemplo n.º 10
0
/* u8x16 mv mul */
void mw_neon_mv_mul_u8x16(unsigned char * A, int Row, int T, unsigned char * B, unsigned char * C)
{
	int i = 0;
	int k = 0;

	uint8x16_t neon_b, neon_c;
	uint8x16_t neon_a0, neon_a1, neon_a2, neon_a3, neon_a4, neon_a5, neon_a6, neon_a7;
	uint8x16_t neon_a8, neon_a9, neon_a10, neon_a11, neon_a12, neon_a13, neon_a14, neon_a15;
	uint8x16_t neon_b0, neon_b1, neon_b2, neon_b3, neon_b4, neon_b5, neon_b6, neon_b7;
	uint8x16_t neon_b8, neon_b9, neon_b10, neon_b11, neon_b12, neon_b13, neon_b14, neon_b15;

	for (i = 0; i < Row; i+=16)
	{
		neon_c = vmovq_n_u8(0);

		for (k = 0; k < T; k+=16)
		{
			int j = k * T + i;

			neon_a0 = vld1q_u8(A + j);
			j+=Row;
			neon_a1 = vld1q_u8(A + j);
			j+=Row;
			neon_a2 = vld1q_u8(A + j);
			j+=Row;
			neon_a3 = vld1q_u8(A + j);
			j+=Row;
			neon_a4 = vld1q_u8(A + j);
			j+=Row;
			neon_a5 = vld1q_u8(A + j);
			j+=Row;
			neon_a6 = vld1q_u8(A + j);
			j+=Row;
			neon_a7 = vld1q_u8(A + j);
			j+=Row;
			neon_a8 = vld1q_u8(A + j);
			j+=Row;
			neon_a9 = vld1q_u8(A + j);
			j+=Row;
			neon_a10 = vld1q_u8(A + j);
			j+=Row;
			neon_a11 = vld1q_u8(A + j);
			j+=Row;
			neon_a12 = vld1q_u8(A + j);
			j+=Row;
			neon_a13 = vld1q_u8(A + j);
			j+=Row;
			neon_a14 = vld1q_u8(A + j);
			j+=Row;
			neon_a15 = vld1q_u8(A + j);

			neon_b = vld1q_u8(B + k);
			neon_b0 = vdupq_n_u8(vgetq_lane_u8(neon_b, 0));
			neon_b1 = vdupq_n_u8(vgetq_lane_u8(neon_b, 1));
			neon_b2 = vdupq_n_u8(vgetq_lane_u8(neon_b, 2));
			neon_b3 = vdupq_n_u8(vgetq_lane_u8(neon_b, 3));
			neon_b4 = vdupq_n_u8(vgetq_lane_u8(neon_b, 4));
			neon_b5 = vdupq_n_u8(vgetq_lane_u8(neon_b, 5));
			neon_b6 = vdupq_n_u8(vgetq_lane_u8(neon_b, 6));
			neon_b7 = vdupq_n_u8(vgetq_lane_u8(neon_b, 7));
			neon_b8 = vdupq_n_u8(vgetq_lane_u8(neon_b, 8));
			neon_b9 = vdupq_n_u8(vgetq_lane_u8(neon_b, 9));
			neon_b10 = vdupq_n_u8(vgetq_lane_u8(neon_b, 10));
			neon_b11 = vdupq_n_u8(vgetq_lane_u8(neon_b, 11));
			neon_b12 = vdupq_n_u8(vgetq_lane_u8(neon_b, 12));
			neon_b13 = vdupq_n_u8(vgetq_lane_u8(neon_b, 13));
			neon_b14 = vdupq_n_u8(vgetq_lane_u8(neon_b, 14));
			neon_b15 = vdupq_n_u8(vgetq_lane_u8(neon_b, 15));

			neon_c = vaddq_u8(vmulq_u8(neon_a0, neon_b0), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a1, neon_b1), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a2, neon_b2), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a3, neon_b3), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a4, neon_b4), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a5, neon_b5), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a6, neon_b6), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a7, neon_b7), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a8, neon_b8), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a9, neon_b9), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a10, neon_b10), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a11, neon_b11), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a12, neon_b12), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a13, neon_b13), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a14, neon_b14), neon_c);
			neon_c = vaddq_u8(vmulq_u8(neon_a15, neon_b15), neon_c);

		}

		vst1q_u8(C + i, neon_c);
	}
}
Ejemplo n.º 11
0
inline  uint8x16_t vaddq(const uint8x16_t  & v0, const uint8x16_t  & v1) { return vaddq_u8 (v0, v1); }
Ejemplo n.º 12
0
size_t mempopcnt(const void *s, size_t len)
{
	uint8x16_t v_0;
	uint8x16_t c;
	uint32x4_t v_sum;
	uint32x2_t v_tsum;
	unsigned char *p;
	size_t r;
	unsigned shift;

	prefetch(s);

// TODO: do this in 64 bit? the mem model seems more that way...
	v_0   = (uint8x16_t){0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
	v_sum = (uint32x4_t)v_0;
	p = (unsigned char *)ALIGN_DOWN(s, SOVUCQ);
	shift = ALIGN_DOWN_DIFF(s, SOVUCQ);
	c = *(const uint8x16_t *)p;
	if(HOST_IS_BIGENDIAN)
		c = neon_simple_alignq(v_0, c, SOVUCQ - shift);
	else
		c = neon_simple_alignq(c, v_0, shift);
	if(len >= SOVUCQ || len + shift >= SOVUCQ)
	{
		p    += SOVUCQ;
		len  -= SOVUCQ - shift;
		v_sum = vpadalq_u16(v_sum, vpaddlq_u8(vcntq_u8(c)));

		while(len >= SOVUCQ * 2)
		{
			uint8x16_t v_sumb = v_0;

			r    = len / (SOVUCQ * 2);
			r    = r > 15 ? 15 : r;
			len -= r * SOVUCQ * 2;
			/*
			 * NEON has a vector popcnt instruction, so no compression.
			 * We trust the speed given in the handbook (adding more
			 * instructions would not make it faster), 1-2 cycles.
			 */
			for(; r; r--, p += SOVUCQ * 2) {
				c      = *(const uint8x16_t *)p;
				v_sumb = vaddq_u8(v_sumb, vcntq_u8(c));
				c      = *((const uint8x16_t *)(p + SOVUCQ));
				v_sumb = vaddq_u8(v_sumb, vcntq_u8(c));
			}
			v_sum = vpadalq_u16(v_sum, vpaddlq_u8(v_sumb));
		}
		if(len >= SOVUCQ) {
			c     = *(const uint8x16_t *)p;
			p    += SOVUCQ;
			v_sum = vpadalq_u16(v_sum, vpaddlq_u8(vcntq_u8(c)));
			len  -= SOVUCQ;
		}

		if(len)
			c = *(const uint8x16_t *)p;
	}
	if(len) {
		if(HOST_IS_BIGENDIAN)
			c      = neon_simple_alignq(c, v_0, SOVUCQ - len);
		else
			c      = neon_simple_alignq(v_0, c, len);
		v_sum  = vpadalq_u16(v_sum, vpaddlq_u8(vcntq_u8(c)));
	}

	v_tsum = vpadd_u32(vget_high_u32(v_sum), vget_low_u32(v_sum));
	v_tsum = vpadd_u32(v_tsum, v_tsum);
	return vget_lane_u32(v_tsum, 0);
}
Ejemplo n.º 13
0
static inline uint8x16x4_t
enc_translate (uint8x16x4_t in)
{
	uint8x16x4_t mask1, mask2, mask3, mask4, out;

	// Translate values 0..63 to the Base64 alphabet. There are five sets:
	// #  From      To         Abs  Delta  Characters
	// 0  [0..25]   [65..90]   +65  +65    ABCDEFGHIJKLMNOPQRSTUVWXYZ
	// 1  [26..51]  [97..122]  +71   +6    abcdefghijklmnopqrstuvwxyz
	// 2  [52..61]  [48..57]    -4  -75    0123456789
	// 3  [62]      [43]       -19  -15    +
	// 4  [63]      [47]       -16   +3    /

	// Create cumulative masks for characters in sets [1,2,3,4], [2,3,4],
	// [3,4], and [4]:
	mask1.val[0] = CMPGT(in.val[0], 25);
	mask1.val[1] = CMPGT(in.val[1], 25);
	mask1.val[2] = CMPGT(in.val[2], 25);
	mask1.val[3] = CMPGT(in.val[3], 25);

	mask2.val[0] = CMPGT(in.val[0], 51);
	mask2.val[1] = CMPGT(in.val[1], 51);
	mask2.val[2] = CMPGT(in.val[2], 51);
	mask2.val[3] = CMPGT(in.val[3], 51);

	mask3.val[0] = CMPGT(in.val[0], 61);
	mask3.val[1] = CMPGT(in.val[1], 61);
	mask3.val[2] = CMPGT(in.val[2], 61);
	mask3.val[3] = CMPGT(in.val[3], 61);

	mask4.val[0] = CMPEQ(in.val[0], 63);
	mask4.val[1] = CMPEQ(in.val[1], 63);
	mask4.val[2] = CMPEQ(in.val[2], 63);
	mask4.val[3] = CMPEQ(in.val[3], 63);

	// All characters are at least in cumulative set 0, so add 'A':
	out.val[0] = vaddq_u8(in.val[0], vdupq_n_u8(65));
	out.val[1] = vaddq_u8(in.val[1], vdupq_n_u8(65));
	out.val[2] = vaddq_u8(in.val[2], vdupq_n_u8(65));
	out.val[3] = vaddq_u8(in.val[3], vdupq_n_u8(65));

	// For inputs which are also in any of the other cumulative sets,
	// add delta values against the previous set(s) to correct the shift:
	out.val[0] = vaddq_u8(out.val[0], REPLACE(mask1.val[0], 6));
	out.val[1] = vaddq_u8(out.val[1], REPLACE(mask1.val[1], 6));
	out.val[2] = vaddq_u8(out.val[2], REPLACE(mask1.val[2], 6));
	out.val[3] = vaddq_u8(out.val[3], REPLACE(mask1.val[3], 6));

	out.val[0] = vsubq_u8(out.val[0], REPLACE(mask2.val[0], 75));
	out.val[1] = vsubq_u8(out.val[1], REPLACE(mask2.val[1], 75));
	out.val[2] = vsubq_u8(out.val[2], REPLACE(mask2.val[2], 75));
	out.val[3] = vsubq_u8(out.val[3], REPLACE(mask2.val[3], 75));

	out.val[0] = vsubq_u8(out.val[0], REPLACE(mask3.val[0], 15));
	out.val[1] = vsubq_u8(out.val[1], REPLACE(mask3.val[1], 15));
	out.val[2] = vsubq_u8(out.val[2], REPLACE(mask3.val[2], 15));
	out.val[3] = vsubq_u8(out.val[3], REPLACE(mask3.val[3], 15));

	out.val[0] = vaddq_u8(out.val[0], REPLACE(mask4.val[0], 3));
	out.val[1] = vaddq_u8(out.val[1], REPLACE(mask4.val[1], 3));
	out.val[2] = vaddq_u8(out.val[2], REPLACE(mask4.val[2], 3));
	out.val[3] = vaddq_u8(out.val[3], REPLACE(mask4.val[3], 3));

	return out;
}