TheTest & test_interleave()
{
Data<R> data1, data2, data3, data4;
data2 += 20;
data3 += 40;
data4 += 60;
R a = data1, b = data2, c = data3;
R d = data1, e = data2, f = data3, g = data4;
LaneType buf3[R::nlanes * 3];
LaneType buf4[R::nlanes * 4];
v_store_interleave(buf3, a, b, c);
v_store_interleave(buf4, d, e, f, g);
Data<R> z(0);
a = b = c = d = e = f = g = z;
v_load_deinterleave(buf3, a, b, c);
v_load_deinterleave(buf4, d, e, f, g);
for (int i = 0; i < R::nlanes; ++i)
{
EXPECT_EQ(data1, Data<R>(a));
EXPECT_EQ(data2, Data<R>(b));
EXPECT_EQ(data3, Data<R>(c));
EXPECT_EQ(data1, Data<R>(d));
EXPECT_EQ(data2, Data<R>(e));
EXPECT_EQ(data3, Data<R>(f));
EXPECT_EQ(data4, Data<R>(g));
}
return *this;
}
// float32x4 only
TheTest & test_interleave_2channel()
{
Data<R> data1, data2;
data2 += 20;
R a = data1, b = data2;
LaneType buf2[R::nlanes * 2];
v_store_interleave(buf2, a, b);
Data<R> z(0);
a = b = z;
v_load_deinterleave(buf2, a, b);
for (int i = 0; i < R::nlanes; ++i)
{
EXPECT_EQ(data1, Data<R>(a));
EXPECT_EQ(data2, Data<R>(b));
}
return *this;
}
/*
The trick with STORE_UNALIGNED/STORE_ALIGNED_NOCACHE is the following:
on IA there are instructions movntps and such to which
v_store_interleave(...., STORE_ALIGNED_NOCACHE) is mapped.
Those instructions write directly into memory w/o touching cache
that results in dramatic speed improvements, especially on
large arrays (FullHD, 4K etc.).
Those intrinsics require the destination address to be aligned
by 16/32 bits (with SSE2 and AVX2, respectively).
So we potentially split the processing into 3 stages:
1) the optional prefix part [0:i0), where we use simple unaligned stores.
2) the optional main part [i0:len - VECSZ], where we use "nocache" mode.
But in some cases we have to use unaligned stores in this part.
3) the optional suffix part (the tail) (len - VECSZ:len) where we switch back to "unaligned" mode
to process the remaining len - VECSZ elements.
In principle there can be very poorly aligned data where there is no main part.
For that we set i0=0 and use unaligned stores for the whole array.
*/
template<typename T, typename VecT> static void
vecmerge_( const T** src, T* dst, int len, int cn )
{
const int VECSZ = VecT::nlanes;
int i, i0 = 0;
const T* src0 = src[0];
const T* src1 = src[1];
const int dstElemSize = cn * sizeof(T);
int r = (int)((size_t)(void*)dst % (VECSZ*sizeof(T)));
hal::StoreMode mode = hal::STORE_ALIGNED_NOCACHE;
if( r != 0 )
{
mode = hal::STORE_UNALIGNED;
if (r % dstElemSize == 0 && len > VECSZ*2)
i0 = VECSZ - (r / dstElemSize);
}
if( cn == 2 )
{
for( i = 0; i < len; i += VECSZ )
{
if( i > len - VECSZ )
{
i = len - VECSZ;
mode = hal::STORE_UNALIGNED;
}
VecT a = vx_load(src0 + i), b = vx_load(src1 + i);
v_store_interleave(dst + i*cn, a, b, mode);
if( i < i0 )
{
i = i0 - VECSZ;
mode = hal::STORE_ALIGNED_NOCACHE;
}
}
}
else if( cn == 3 )
{
const T* src2 = src[2];
for( i = 0; i < len; i += VECSZ )
{
if( i > len - VECSZ )
{
i = len - VECSZ;
mode = hal::STORE_UNALIGNED;
}
VecT a = vx_load(src0 + i), b = vx_load(src1 + i), c = vx_load(src2 + i);
v_store_interleave(dst + i*cn, a, b, c, mode);
if( i < i0 )
{
i = i0 - VECSZ;
mode = hal::STORE_ALIGNED_NOCACHE;
}
}
}
else
{
CV_Assert( cn == 4 );
const T* src2 = src[2];
const T* src3 = src[3];
for( i = 0; i < len; i += VECSZ )
{
if( i > len - VECSZ )
{
i = len - VECSZ;
mode = hal::STORE_UNALIGNED;
}
VecT a = vx_load(src0 + i), b = vx_load(src1 + i);
VecT c = vx_load(src2 + i), d = vx_load(src3 + i);
v_store_interleave(dst + i*cn, a, b, c, d, mode);
if( i < i0 )
{
i = i0 - VECSZ;
mode = hal::STORE_ALIGNED_NOCACHE;
}
}
}
vx_cleanup();
}
