int norx_aead_decrypt( unsigned char *m, size_t *mlen, const unsigned char *a, size_t alen, const unsigned char *c, size_t clen, const unsigned char *z, size_t zlen, const unsigned char *nonce, const unsigned char *key ) { const uint32x4_t K = LOADU(key); uint32x4_t A, B, C, D; if(clen < BYTES(NORX_T)) { return -1; } *mlen = clen - BYTES(NORX_T); INITIALISE(A, B, C, D, nonce, K); ABSORB_DATA(A, B, C, D, a, alen, HEADER_TAG); DECRYPT_DATA(A, B, C, D, m, c, clen - BYTES(NORX_T)); ABSORB_DATA(A, B, C, D, z, zlen, TRAILER_TAG); FINALISE(A, B, C, D, K); /* Verify tag */ D = vceqq_u32(D, LOADU(c + clen - BYTES(NORX_T))); return 0xFFFFFFFF == (vgetq_lane_u32(D, 0) & vgetq_lane_u32(D, 1) & vgetq_lane_u32(D, 2) & vgetq_lane_u32(D, 3)) ? 0 : -1; }
/* u32x4 mm mul */ void mw_neon_mm_mul_u32x4(unsigned int * A, int Row, int T, unsigned int * B, int Col, unsigned int * C) { int i, k, j; uint32x4_t neon_b, neon_c; uint32x4_t neon_a0, neon_a1, neon_a2, neon_a3; uint32x4_t neon_b0, neon_b1, neon_b2, neon_b3; for (i = 0; i < Row; i+=4) { for (k = 0; k < Col; k+=1) { neon_c = vmovq_n_u32(0); for (j = 0; j < T; j+=4) { int j_T = j * T + i; int k_Row = k * Row; neon_a0 = vld1q_u32(A + j_T); j_T+=Row; neon_a1 = vld1q_u32(A + j_T); j_T+=Row; neon_a2 = vld1q_u32(A + j_T); j_T+=Row; neon_a3 = vld1q_u32(A + j_T); neon_b = vld1q_u32(B + k_Row + j); neon_b0 = vdupq_n_u32(vgetq_lane_u32(neon_b, 0)); neon_b1 = vdupq_n_u32(vgetq_lane_u32(neon_b, 1)); neon_b2 = vdupq_n_u32(vgetq_lane_u32(neon_b, 2)); neon_b3 = vdupq_n_u32(vgetq_lane_u32(neon_b, 3)); neon_c = vaddq_u32(vmulq_u32(neon_a0, neon_b0), neon_c); neon_c = vaddq_u32(vmulq_u32(neon_a1, neon_b1), neon_c); neon_c = vaddq_u32(vmulq_u32(neon_a2, neon_b2), neon_c); neon_c = vaddq_u32(vmulq_u32(neon_a3, neon_b3), neon_c); vst1q_lane_u32(C + k_Row + i, neon_c, 0); vst1q_lane_u32(C + k_Row + i + 1, neon_c, 1); vst1q_lane_u32(C + k_Row + i + 2, neon_c, 2); vst1q_lane_u32(C + k_Row + i + 3, neon_c, 3); } } } }
/* u32x4 mv mul */ void mw_neon_mv_mul_u32x4(unsigned int * A, int Row, int T, unsigned int * B, unsigned int * C) { int i = 0; int k = 0; uint32x4_t neon_b, neon_c; uint32x4_t neon_a0, neon_a1, neon_a2, neon_a3; uint32x4_t neon_b0, neon_b1, neon_b2, neon_b3; for (i = 0; i < Row; i+=4) { neon_c = vmovq_n_u32(0); for (k = 0; k < T; k+=4) { int j = k * T + i; neon_a0 = vld1q_u32(A + j); j+=Row; neon_a1 = vld1q_u32(A + j); j+=Row; neon_a2 = vld1q_u32(A + j); j+=Row; neon_a3 = vld1q_u32(A + j); neon_b = vld1q_u32(B + k); neon_b0 = vdupq_n_u32(vgetq_lane_u32(neon_b, 0)); neon_b1 = vdupq_n_u32(vgetq_lane_u32(neon_b, 1)); neon_b2 = vdupq_n_u32(vgetq_lane_u32(neon_b, 2)); neon_b3 = vdupq_n_u32(vgetq_lane_u32(neon_b, 3)); neon_c = vaddq_u32(vmulq_u32(neon_a0, neon_b0), neon_c); neon_c = vaddq_u32(vmulq_u32(neon_a1, neon_b1), neon_c); neon_c = vaddq_u32(vmulq_u32(neon_a2, neon_b2), neon_c); neon_c = vaddq_u32(vmulq_u32(neon_a3, neon_b3), neon_c); } vst1q_u32(C + i, neon_c); } }
uint32_t test_vgetq_lane_u32(uint32x4_t v1) { // CHECK: test_vgetq_lane_u32 return vgetq_lane_u32(v1, 2); // CHECK: umov {{w[0-9]+}}, {{v[0-9]+}}.s[2] }
static inline void desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4], struct rte_mbuf **rx_pkts) { uint32x4_t vlan0, vlan1, rss, l3_l4e; const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0}; uint64x2_t rearm0, rearm1, rearm2, rearm3; /* mask everything except RSS, flow director and VLAN flags * bit2 is for VLAN tag, bit11 for flow director indication * bit13:12 for RSS indication. */ const uint32x4_t rss_vlan_msk = { 0x1c03804, 0x1c03804, 0x1c03804, 0x1c03804}; const uint32x4_t cksum_mask = { PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | PKT_RX_EIP_CKSUM_BAD, PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | PKT_RX_EIP_CKSUM_BAD, PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | PKT_RX_EIP_CKSUM_BAD, PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | PKT_RX_EIP_CKSUM_BAD}; /* map rss and vlan type to rss hash and vlan flag */ const uint8x16_t vlan_flags = { 0, 0, 0, 0, PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; const uint8x16_t rss_flags = { 0, PKT_RX_FDIR, 0, 0, 0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH | PKT_RX_FDIR, 0, 0, 0, 0, 0, 0, 0, 0}; const uint8x16_t l3_l4e_flags = { (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1, PKT_RX_IP_CKSUM_BAD >> 1, (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1, (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD) >> 1, (PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD) >> 1, (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, 0, 0, 0, 0, 0, 0, 0, 0}; vlan0 = vzipq_u32(vreinterpretq_u32_u64(descs[0]), vreinterpretq_u32_u64(descs[2])).val[1]; vlan1 = vzipq_u32(vreinterpretq_u32_u64(descs[1]), vreinterpretq_u32_u64(descs[3])).val[1]; vlan0 = vzipq_u32(vlan0, vlan1).val[0]; vlan1 = vandq_u32(vlan0, rss_vlan_msk); vlan0 = vreinterpretq_u32_u8(vqtbl1q_u8(vlan_flags, vreinterpretq_u8_u32(vlan1))); rss = vshrq_n_u32(vlan1, 11); rss = vreinterpretq_u32_u8(vqtbl1q_u8(rss_flags, vreinterpretq_u8_u32(rss))); l3_l4e = vshrq_n_u32(vlan1, 22); l3_l4e = vreinterpretq_u32_u8(vqtbl1q_u8(l3_l4e_flags, vreinterpretq_u8_u32(l3_l4e))); /* then we shift left 1 bit */ l3_l4e = vshlq_n_u32(l3_l4e, 1); /* we need to mask out the reduntant bits */ l3_l4e = vandq_u32(l3_l4e, cksum_mask); vlan0 = vorrq_u32(vlan0, rss); vlan0 = vorrq_u32(vlan0, l3_l4e); rearm0 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 0), mbuf_init, 1); rearm1 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 1), mbuf_init, 1); rearm2 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 2), mbuf_init, 1); rearm3 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 3), mbuf_init, 1); vst1q_u64((uint64_t *)&rx_pkts[0]->rearm_data, rearm0); vst1q_u64((uint64_t *)&rx_pkts[1]->rearm_data, rearm1); vst1q_u64((uint64_t *)&rx_pkts[2]->rearm_data, rearm2); vst1q_u64((uint64_t *)&rx_pkts[3]->rearm_data, rearm3); }
uint32_t test_vgetq_lane_u32(uint32x4_t a) { // CHECK-LABEL: test_vgetq_lane_u32: // CHECK-NEXT: mov.s w0, v0[3] // CHECK-NEXT: ret return vgetq_lane_u32(a, 3); }
bool CPU_ProbeNEON() { #if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES) return false; #elif (CRYPTOPP_ARM_NEON_AVAILABLE) # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY) volatile bool result = true; __try { uint32_t v1[4] = {1,1,1,1}; uint32x4_t x1 = vld1q_u32(v1); uint64_t v2[2] = {1,1}; uint64x2_t x2 = vld1q_u64(v2); uint32x4_t x3 = vdupq_n_u32(2); x3 = vsetq_lane_u32(vgetq_lane_u32(x1,0),x3,0); x3 = vsetq_lane_u32(vgetq_lane_u32(x1,3),x3,3); uint64x2_t x4 = vdupq_n_u64(2); x4 = vsetq_lane_u64(vgetq_lane_u64(x2,0),x4,0); x4 = vsetq_lane_u64(vgetq_lane_u64(x2,1),x4,1); result = !!(vgetq_lane_u32(x3,0) | vgetq_lane_u64(x4,1)); } __except (EXCEPTION_EXECUTE_HANDLER) { return false; } return result; # else // longjmp and clobber warnings. Volatile is required. // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854 volatile bool result = true; volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler); if (oldHandler == SIG_ERR) return false; volatile sigset_t oldMask; if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask)) return false; if (setjmp(s_jmpSIGILL)) result = false; else { uint32_t v1[4] = {1,1,1,1}; uint32x4_t x1 = vld1q_u32(v1); uint64_t v2[2] = {1,1}; uint64x2_t x2 = vld1q_u64(v2); uint32x4_t x3 = {0,0,0,0}; x3 = vsetq_lane_u32(vgetq_lane_u32(x1,0),x3,0); x3 = vsetq_lane_u32(vgetq_lane_u32(x1,3),x3,3); uint64x2_t x4 = {0,0}; x4 = vsetq_lane_u64(vgetq_lane_u64(x2,0),x4,0); x4 = vsetq_lane_u64(vgetq_lane_u64(x2,1),x4,1); // Hack... GCC optimizes away the code and returns true result = !!(vgetq_lane_u32(x3,0) | vgetq_lane_u64(x4,1)); } sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR); signal(SIGILL, oldHandler); return result; # endif #else return false; #endif // CRYPTOPP_ARM_NEON_AVAILABLE }
// CHECK-LABEL: define i32 @test_vgetq_lane_u32(<4 x i32> %a) #0 { // CHECK: [[TMP0:%.*]] = bitcast <4 x i32> %a to <16 x i8> // CHECK: [[TMP1:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32> // CHECK: [[VGETQ_LANE:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3 // CHECK: ret i32 [[VGETQ_LANE]] uint32_t test_vgetq_lane_u32(uint32x4_t a) { return vgetq_lane_u32(a, 3); }