namespace Avx2
{
const __m256i K16_BLUE_RED = SIMD_MM256_SET2_EPI16(Base::BLUE_TO_GRAY_WEIGHT, Base::RED_TO_GRAY_WEIGHT);
const __m256i K16_GREEN_ROUND = SIMD_MM256_SET2_EPI16(Base::GREEN_TO_GRAY_WEIGHT, Base::BGR_TO_GRAY_ROUND_TERM);
SIMD_INLINE __m256i BgraToGray32(__m256i bgra)
{
const __m256i g0a0 = _mm256_and_si256(_mm256_srli_si256(bgra, 1), K16_00FF);
const __m256i b0r0 = _mm256_and_si256(bgra, K16_00FF);
const __m256i weightedSum = _mm256_add_epi32(_mm256_madd_epi16(g0a0, K16_GREEN_ROUND), _mm256_madd_epi16(b0r0, K16_BLUE_RED));
return _mm256_srli_epi32(weightedSum, Base::BGR_TO_GRAY_AVERAGING_SHIFT);
}
SIMD_INLINE __m256i BgraToGray(__m256i bgra[4])
{
const __m256i lo = PackI32ToI16(BgraToGray32(bgra[0]), BgraToGray32(bgra[1]));
const __m256i hi = PackI32ToI16(BgraToGray32(bgra[2]), BgraToGray32(bgra[3]));
return PackU16ToU8(lo, hi);
}
template <bool align> SIMD_INLINE __m256i BgrToGray(const uint8_t * bgr)
{
__m256i bgra[4];
bgra[0] = BgrToBgra<false>(Load<align>((__m256i*)(bgr + 0)), K32_01000000);
bgra[1] = BgrToBgra<false>(Load<false>((__m256i*)(bgr + 24)), K32_01000000);
bgra[2] = BgrToBgra<false>(Load<false>((__m256i*)(bgr + 48)), K32_01000000);
bgra[3] = BgrToBgra<true>(Load<align>((__m256i*)(bgr + 64)), K32_01000000);
return BgraToGray(bgra);
}
template <bool align> void BgrToGray(const uint8_t * bgr, size_t width, size_t height, size_t bgrStride, uint8_t * gray, size_t grayStride)
{
assert(width >= A);
if(align)
assert(Aligned(gray) && Aligned(grayStride) && Aligned(bgr) && Aligned(bgrStride));
size_t alignedWidth = AlignLo(width, A);
for(size_t row = 0; row < height; ++row)
{
for(size_t col = 0; col < alignedWidth; col += A)
Store<align>((__m256i*)(gray + col), BgrToGray<align>(bgr + 3*col));
if(width != alignedWidth)
Store<false>((__m256i*)(gray + width - A), BgrToGray<false>(bgr + 3*(width - A)));
bgr += bgrStride;
gray += grayStride;
}
}
void BgrToGray(const uint8_t * bgr, size_t width, size_t height, size_t bgrStride, uint8_t * gray, size_t grayStride)
{
if(Aligned(gray) && Aligned(grayStride) && Aligned(bgr) && Aligned(bgrStride))
BgrToGray<true>(bgr, width, height, bgrStride, gray, grayStride);
else
BgrToGray<false>(bgr, width, height, bgrStride, gray, grayStride);
}
}
namespace Avx2
{
const __m256i K16_BLUE_RED = SIMD_MM256_SET2_EPI16(Base::BLUE_TO_GRAY_WEIGHT, Base::RED_TO_GRAY_WEIGHT);
const __m256i K16_GREEN_0000 = SIMD_MM256_SET2_EPI16(Base::GREEN_TO_GRAY_WEIGHT, 0x0000);
const __m256i K32_ROUND_TERM = SIMD_MM256_SET1_EPI32(Base::BGR_TO_GRAY_ROUND_TERM);
SIMD_INLINE __m256i BgraToGray32(__m256i bgra)
{
const __m256i g0a0 = _mm256_and_si256(_mm256_srli_si256(bgra, 1), K16_00FF);
const __m256i b0r0 = _mm256_and_si256(bgra, K16_00FF);
const __m256i weightedSum = _mm256_add_epi32(_mm256_madd_epi16(g0a0, K16_GREEN_0000), _mm256_madd_epi16(b0r0, K16_BLUE_RED));
return _mm256_srli_epi32(_mm256_add_epi32(weightedSum, K32_ROUND_TERM), Base::BGR_TO_GRAY_AVERAGING_SHIFT);
}
SIMD_INLINE __m256i BgraToGray(__m256i bgra[4])
{
const __m256i lo = PackI32ToI16(BgraToGray32(bgra[0]), BgraToGray32(bgra[1]));
const __m256i hi = PackI32ToI16(BgraToGray32(bgra[2]), BgraToGray32(bgra[3]));
return PackU16ToU8(lo, hi);
}
template <bool align> SIMD_INLINE void Load(const uint8_t* p, __m256i a[4])
{
a[0] = Load<align>((__m256i*)p + 0);
a[1] = Load<align>((__m256i*)p + 1);
a[2] = Load<align>((__m256i*)p + 2);
a[3] = Load<align>((__m256i*)p + 3);
}
template <bool align> void BgraToGray(const uint8_t *bgra, size_t width, size_t height, size_t bgraStride, uint8_t *gray, size_t grayStride)
{
assert(width >= A);
if (align)
assert(Aligned(bgra) && Aligned(bgraStride) && Aligned(gray) && Aligned(grayStride));
size_t alignedWidth = AlignLo(width, A);
__m256i a[4];
for (size_t row = 0; row < height; ++row)
{
for (size_t col = 0; col < alignedWidth; col += A)
{
Load<align>(bgra + 4 * col, a);
Store<align>((__m256i*)(gray + col), BgraToGray(a));
}
if (alignedWidth != width)
{
Load<false>(bgra + 4 * (width - A), a);
Store<false>((__m256i*)(gray + width - A), BgraToGray(a));
}
bgra += bgraStride;
gray += grayStride;
}
}
void BgraToGray(const uint8_t *bgra, size_t width, size_t height, size_t bgraStride, uint8_t *gray, size_t grayStride)
{
if (Aligned(bgra) && Aligned(gray) && Aligned(bgraStride) && Aligned(grayStride))
BgraToGray<true>(bgra, width, height, bgraStride, gray, grayStride);
else
BgraToGray<false>(bgra, width, height, bgraStride, gray, grayStride);
}
}
namespace Avx2
{
const __m256i K16_BLUE_RED = SIMD_MM256_SET2_EPI16(Base::BLUE_TO_GRAY_WEIGHT, Base::RED_TO_GRAY_WEIGHT);
const __m256i K16_GREEN_ROUND = SIMD_MM256_SET2_EPI16(Base::GREEN_TO_GRAY_WEIGHT, Base::BGR_TO_GRAY_ROUND_TERM);
SIMD_INLINE __m256i BgraToGray32(__m256i bgra)
{
const __m256i g0a0 = _mm256_and_si256(_mm256_srli_si256(bgra, 1), K16_00FF);
const __m256i b0r0 = _mm256_and_si256(bgra, K16_00FF);
const __m256i weightedSum = _mm256_add_epi32(_mm256_madd_epi16(g0a0, K16_GREEN_ROUND), _mm256_madd_epi16(b0r0, K16_BLUE_RED));
return _mm256_srli_epi32(weightedSum, Base::BGR_TO_GRAY_AVERAGING_SHIFT);
}
SIMD_INLINE __m256i BgraToGray(__m256i bgra[4])
{
const __m256i lo = PackI32ToI16(BgraToGray32(bgra[0]), BgraToGray32(bgra[1]));
const __m256i hi = PackI32ToI16(BgraToGray32(bgra[2]), BgraToGray32(bgra[3]));
return PackU16ToU8(lo, hi);
}
SIMD_INLINE __m256i PermuteAndShiffle(__m256i bgr, __m256i permute, __m256i shuffle)
{
return _mm256_shuffle_epi8(_mm256_permutevar8x32_epi32(bgr, permute), shuffle);
}
template <bool align> SIMD_INLINE __m256i BgrToGray(const uint8_t * bgr, __m256i permuteBody, __m256i permuteTail, __m256i shuffle)
{
__m256i bgra[4];
bgra[0] = _mm256_or_si256(K32_01000000, PermuteAndShiffle(Load<align>((__m256i*)(bgr + 0)), permuteBody, shuffle));
bgra[1] = _mm256_or_si256(K32_01000000, PermuteAndShiffle(Load<false>((__m256i*)(bgr + 24)), permuteBody, shuffle));
bgra[2] = _mm256_or_si256(K32_01000000, PermuteAndShiffle(Load<false>((__m256i*)(bgr + 48)), permuteBody, shuffle));
bgra[3] = _mm256_or_si256(K32_01000000, PermuteAndShiffle(Load<align>((__m256i*)(bgr + 64)), permuteTail, shuffle));
return BgraToGray(bgra);
}
template <bool align> void BgrToGray(const uint8_t * bgr, size_t width, size_t height, size_t bgrStride, uint8_t * gray, size_t grayStride)
{
assert(width >= A);
if(align)
assert(Aligned(gray) && Aligned(grayStride) && Aligned(bgr) && Aligned(bgrStride));
size_t alignedWidth = AlignLo(width, A);
__m256i _permuteBody = _mm256_setr_epi32(0, 1, 2, 0, 3, 4, 5, 0);
__m256i _permuteTail = _mm256_setr_epi32(2, 3, 4, 0, 5, 6, 7, 0);
__m256i _shuffle = _mm256_setr_epi8(
0x0, 0x1, 0x2, -1, 0x3, 0x4, 0x5, -1, 0x6, 0x7, 0x8, -1, 0x9, 0xA, 0xB, -1,
0x0, 0x1, 0x2, -1, 0x3, 0x4, 0x5, -1, 0x6, 0x7, 0x8, -1, 0x9, 0xA, 0xB, -1);
for(size_t row = 0; row < height; ++row)
{
for(size_t col = 0; col < alignedWidth; col += A)
Store<align>((__m256i*)(gray + col), BgrToGray<align>(bgr + 3*col, _permuteBody, _permuteTail, _shuffle));
if(width != alignedWidth)
Store<false>((__m256i*)(gray + width - A), BgrToGray<false>(bgr + 3*(width - A), _permuteBody, _permuteTail, _shuffle));
bgr += bgrStride;
gray += grayStride;
}
}
void BgrToGray(const uint8_t * bgr, size_t width, size_t height, size_t bgrStride, uint8_t * gray, size_t grayStride)
{
if(Aligned(gray) && Aligned(grayStride) && Aligned(bgr) && Aligned(bgrStride))
BgrToGray<true>(bgr, width, height, bgrStride, gray, grayStride);
else
BgrToGray<false>(bgr, width, height, bgrStride, gray, grayStride);
}
}
