static void yuv2plane1_8_vsx(const int16_t *src, uint8_t *dest, int dstW,
const uint8_t *dither, int offset)
{
const int dst_u = -(uintptr_t)dest & 15;
int i, j;
LOCAL_ALIGNED(16, int16_t, val, [16]);
const vector uint16_t shifts = (vector uint16_t) {7, 7, 7, 7, 7, 7, 7, 7};
vector int16_t vi, vileft, ditherleft, ditherright;
vector uint8_t vd;
for (j = 0; j < 16; j++) {
val[j] = dither[(dst_u + offset + j) & 7];
}
ditherleft = vec_ld(0, val);
ditherright = vec_ld(0, &val[8]);
yuv2plane1_8_u(src, dest, dst_u, dither, offset, 0);
for (i = dst_u; i < dstW - 15; i += 16) {
vi = vec_vsx_ld(0, &src[i]);
vi = vec_adds(ditherleft, vi);
vileft = vec_sra(vi, shifts);
vi = vec_vsx_ld(0, &src[i + 8]);
vi = vec_adds(ditherright, vi);
vi = vec_sra(vi, shifts);
vd = vec_packsu(vileft, vi);
vec_st(vd, 0, &dest[i]);
}
yuv2plane1_8_u(src, dest, dstW, dither, offset, i);
}
static void predict_16x16_p_altivec( uint8_t *src )
{
int16_t a, b, c, i;
int H = 0;
int V = 0;
int16_t i00;
for( i = 1; i <= 8; i++ )
{
H += i * ( src[7+i - FDEC_STRIDE ] - src[7-i - FDEC_STRIDE ] );
V += i * ( src[(7+i)*FDEC_STRIDE -1] - src[(7-i)*FDEC_STRIDE -1] );
}
a = 16 * ( src[15*FDEC_STRIDE -1] + src[15 - FDEC_STRIDE] );
b = ( 5 * H + 32 ) >> 6;
c = ( 5 * V + 32 ) >> 6;
i00 = a - b * 7 - c * 7 + 16;
vect_sshort_u i00_u, b_u, c_u;
i00_u.s[0] = i00;
b_u.s[0] = b;
c_u.s[0] = c;
vec_u16_t val5_v = vec_splat_u16(5);
vec_s16_t i00_v, b_v, c_v;
i00_v = vec_splat(i00_u.v, 0);
b_v = vec_splat(b_u.v, 0);
c_v = vec_splat(c_u.v, 0);
vec_s16_t induc_v = (vec_s16_t) CV(0, 1, 2, 3, 4, 5, 6, 7);
vec_s16_t b8_v = vec_sl(b_v, vec_splat_u16(3));
vec_s32_t mule_b_v = vec_mule(induc_v, b_v);
vec_s32_t mulo_b_v = vec_mulo(induc_v, b_v);
vec_s16_t mul_b_induc0_v = vec_pack(vec_mergeh(mule_b_v, mulo_b_v), vec_mergel(mule_b_v, mulo_b_v));
vec_s16_t add_i0_b_0v = vec_adds(i00_v, mul_b_induc0_v);
vec_s16_t add_i0_b_8v = vec_adds(b8_v, add_i0_b_0v);
int y;
for( y = 0; y < 16; y++ )
{
vec_s16_t shift_0_v = vec_sra(add_i0_b_0v, val5_v);
vec_s16_t shift_8_v = vec_sra(add_i0_b_8v, val5_v);
vec_u8_t com_sat_v = vec_packsu(shift_0_v, shift_8_v);
vec_st( com_sat_v, 0, &src[0]);
src += FDEC_STRIDE;
i00 += c;
add_i0_b_0v = vec_adds(add_i0_b_0v, c_v);
add_i0_b_8v = vec_adds(add_i0_b_8v, c_v);
}
}
static av_always_inline void h264_idct_dc_add_internal(uint8_t *dst, int16_t *block, int stride, int size)
{
vec_s16 dc16;
vec_u8 dcplus, dcminus, v0, v1, v2, v3, aligner;
vec_s32 v_dc32;
LOAD_ZERO;
DECLARE_ALIGNED(16, int, dc);
int i;
dc = (block[0] + 32) >> 6;
block[0] = 0;
v_dc32 = vec_lde(0, &dc);
dc16 = VEC_SPLAT16((vec_s16)v_dc32, 1);
if (size == 4)
dc16 = VEC_SLD16(dc16, zero_s16v, 8);
dcplus = vec_packsu(dc16, zero_s16v);
dcminus = vec_packsu(vec_sub(zero_s16v, dc16), zero_s16v);
aligner = vec_lvsr(0, dst);
#if !HAVE_BIGENDIAN
aligner = vec_perm(aligner, zero_u8v, vcswapc());
#endif
dcplus = vec_perm(dcplus, dcplus, aligner);
dcminus = vec_perm(dcminus, dcminus, aligner);
for (i = 0; i < size; i += 4) {
v0 = vec_ld(0, dst+0*stride);
v1 = vec_ld(0, dst+1*stride);
v2 = vec_ld(0, dst+2*stride);
v3 = vec_ld(0, dst+3*stride);
v0 = vec_adds(v0, dcplus);
v1 = vec_adds(v1, dcplus);
v2 = vec_adds(v2, dcplus);
v3 = vec_adds(v3, dcplus);
v0 = vec_subs(v0, dcminus);
v1 = vec_subs(v1, dcminus);
v2 = vec_subs(v2, dcminus);
v3 = vec_subs(v3, dcminus);
vec_st(v0, 0, dst+0*stride);
vec_st(v1, 0, dst+1*stride);
vec_st(v2, 0, dst+2*stride);
vec_st(v3, 0, dst+3*stride);
dst += 4*stride;
}
}
/* *************************************************************************
NAME: test_add_subtract
USAGE:
test_add_subtract();
returns: void
DESCRIPTION:
see how add and subtract work on vectors
REFERENCES:
Ian Ollmann's Altivec Tutorial
LIMITATIONS:
GLOBAL VARIABLES:
accessed: none
modified: none
FUNCTIONS CALLED:
fprintf
vec_add - add two vectors
vec_adds - add two vectors, saturation
vec_sub - subtract two vectors
REVISION HISTORY:
STR Description of Revision Author
27-Feb-11 initial coding kaj
************************************************************************* */
void test_add_subtract(void)
{
vector unsigned short addVector1 =
{ 0, 1000, 5000, 10000, 15000, 20000, 50000, 65535};
vector signed short addSVector1 =
{ -32768, -10000, -5000, 0, 10, 5000, 10000, 32767};
vector signed short addSVector2 =
{ -10, -10, -10, 0, 10, 10, 10, 10};
vector unsigned short sumVector;
vector signed short sumSVector;
short printshort[SHORT_ARRAYSIZE] __attribute__ ((aligned (16)));
/* vec_add should wrap, vec_adds will chop at max/min */
fprintf(stderr,"-----------------------------------------------------------\n");
/* add 10 to each element - unsigned short */
printVecUShorts("vec_add unsigned short input vector", addVector1,SHORT_ARRAYSIZE);
sumVector = vec_add(addVector1, vec_splat_u16(10));
printVecUShorts("vec_add sum vector (Input+10) ", sumVector,SHORT_ARRAYSIZE);
/* add 10 to each element using saturation add - unsigned short */
sumVector = vec_adds(addVector1, vec_splat_u16(10));
printVecUShorts("vec_adds sum vector (Input+10)", sumVector,SHORT_ARRAYSIZE);
/* subtract 10 from each element - unsigned short */
sumVector = vec_sub(addVector1, vec_splat_u16(10));
printVecUShorts("vec_sub sum vector (Input-10) ", sumVector,SHORT_ARRAYSIZE);
fprintf(stderr,"-----------------------------------------------------------\n\n");
/* add 10 to each element - signed short */
printVecShorts("vec_add signed short input vector", addSVector1,SHORT_ARRAYSIZE);
sumSVector = vec_add(addSVector1,addSVector2);
printVecShorts("vec_add sum vector (increment pos & neg by 10) ",
sumSVector,SHORT_ARRAYSIZE);
/* add 10 to each element using saturation add - signed short */
sumSVector = vec_adds(addSVector1,addSVector2);
printVecShorts("vec_adds sum vector (increment pos & neg by 10)",
sumSVector,SHORT_ARRAYSIZE);
/* subtract 10 from each element - signed short */
sumSVector = vec_sub(addSVector1,addSVector2);
printVecShorts("vec_sub vector (decrement pos & neg by 10) ",
sumSVector,SHORT_ARRAYSIZE);
} /* test_add_subtract */
void
gimp_composite_difference_rgba8_rgba8_rgba8_altivec (GimpCompositeContext *ctx)
{
const guchar *A = ctx->A;
const guchar *B = ctx->B;
guchar *D = ctx->D;
guint length = ctx->n_pixels;
vector unsigned char a,b,d,e,alpha_a,alpha_b;
while (length >= 4)
{
a=LoadUnaligned(A);
b=LoadUnaligned(B);
alpha_a=vec_and(a, alphamask);
alpha_b=vec_and(b, alphamask);
d=vec_min(alpha_a, alpha_b);
a=vec_andc(a, alphamask);
a=vec_adds(a, d);
b=vec_andc(b, alphamask);
d=vec_subs(a, b);
e=vec_subs(b, a);
d=vec_add(d,e);
StoreUnaligned(d, D);
A+=16;
B+=16;
D+=16;
length-=4;
}
/* process last pixels */
length = length*4;
a=LoadUnalignedLess(A, length);
b=LoadUnalignedLess(B, length);
alpha_a=vec_and(a,alphamask);
alpha_b=vec_and(b,alphamask);
d=vec_min(alpha_a,alpha_b);
a=vec_andc(a,alphamask);
a=vec_adds(a,d);
b=vec_andc(b,alphamask);
d=vec_subs(a,b);
e=vec_subs(b, a);
d=vec_add(d,e);
StoreUnalignedLess(d, D, length);
}
// out: newp1 = clip((p2 + ((p0 + q0 + 1) >> 1)) >> 1, p1-tc0, p1+tc0)
static inline vec_u8_t h264_deblock_q1(register vec_u8_t p0,
register vec_u8_t p1,
register vec_u8_t p2,
register vec_u8_t q0,
register vec_u8_t tc0) {
register vec_u8_t average = vec_avg(p0, q0);
register vec_u8_t temp;
register vec_u8_t uncliped;
register vec_u8_t ones;
register vec_u8_t max;
register vec_u8_t min;
register vec_u8_t newp1;
temp = vec_xor(average, p2);
average = vec_avg(average, p2); /*avg(p2, avg(p0, q0)) */
ones = vec_splat_u8(1);
temp = vec_and(temp, ones); /*(p2^avg(p0, q0)) & 1 */
uncliped = vec_subs(average, temp); /*(p2+((p0+q0+1)>>1))>>1 */
max = vec_adds(p1, tc0);
min = vec_subs(p1, tc0);
newp1 = vec_max(min, uncliped);
newp1 = vec_min(max, newp1);
return newp1;
}
// out: newp1 = clip((p2 + ((p0 + q0 + 1) >> 1)) >> 1, p1-tc0, p1+tc0)
static inline vector unsigned char h264_deblock_q1(register vector unsigned char p0,
register vector unsigned char p1,
register vector unsigned char p2,
register vector unsigned char q0,
register vector unsigned char tc0) {
register vector unsigned char average = vec_avg(p0, q0);
register vector unsigned char temp;
register vector unsigned char uncliped;
register vector unsigned char ones;
register vector unsigned char max;
register vector unsigned char min;
register vector unsigned char newp1;
temp = vec_xor(average, p2);
average = vec_avg(average, p2); /*avg(p2, avg(p0, q0)) */
ones = vec_splat_u8(1);
temp = vec_and(temp, ones); /*(p2^avg(p0, q0)) & 1 */
uncliped = vec_subs(average, temp); /*(p2+((p0+q0+1)>>1))>>1 */
max = vec_adds(p1, tc0);
min = vec_subs(p1, tc0);
newp1 = vec_max(min, uncliped);
newp1 = vec_min(max, newp1);
return newp1;
}
void pix_add :: processRGBA_Altivec(imageStruct &image, imageStruct &right)
{
int h,w,width;
width = image.xsize/4;
vector unsigned char *inData = (vector unsigned char*) image.data;
vector unsigned char *rightData = (vector unsigned char*) right.data;
#ifndef PPC970
UInt32 prefetchSize = GetPrefetchConstant( 16, 1, 256 );
vec_dst( inData, prefetchSize, 0 );
vec_dst( rightData, prefetchSize, 1 );
#endif
for ( h=0; h<image.ysize; h++){
for (w=0; w<width; w++)
{
#ifndef PPC970
vec_dst( inData, prefetchSize, 0 );
vec_dst( rightData, prefetchSize, 1 );
#endif
inData[0] = vec_adds(inData[0], rightData[0]);
inData++;
rightData++;
}
#ifndef PPC970
vec_dss( 0 );
vec_dss( 1 );
#endif
} /*end of working altivec function */
}
void imageFilterMean_Altivec(unsigned char *src1, unsigned char *src2, unsigned char *dst, int length)
{
int n = length;
// Compute first few values so we're on a 16-byte boundary in dst
while( (((long)dst & 0xF) > 0) && (n > 0) ) {
MEAN_PIXEL();
--n; ++dst; ++src1; ++src2;
}
// Do bulk of processing using Altivec (find the mean of 16 8-bit unsigned integers, with saturation)
vector unsigned char rshft = vec_splat_u8(0x1);
while(n >= 16) {
vector unsigned char s1 = vec_ld(0,src1);
s1 = vec_sr(s1, rshft); // shift right 1
vector unsigned char s2 = vec_ld(0,src2);
s2 = vec_sr(s2, rshft); // shift right 1
vector unsigned char r = vec_adds(s1, s2);
vec_st(r,0,dst);
n -= 16; src1 += 16; src2 += 16; dst += 16;
}
// If any bytes are left over, deal with them individually
++n;
BASIC_MEAN();
}
static av_always_inline void h264_idct_dc_add_internal(uint8_t *dst, int16_t *block, int stride, int size)
{
vec_s16 dc16;
vec_u8 dcplus, dcminus, v0, v1, v2, v3, aligner;
LOAD_ZERO;
DECLARE_ALIGNED(16, int, dc);
int i;
dc = (block[0] + 32) >> 6;
block[0] = 0;
dc16 = vec_splat((vec_s16) vec_lde(0, &dc), 1);
if (size == 4)
dc16 = vec_sld(dc16, zero_s16v, 8);
dcplus = vec_packsu(dc16, zero_s16v);
dcminus = vec_packsu(vec_sub(zero_s16v, dc16), zero_s16v);
aligner = vec_lvsr(0, dst);
dcplus = vec_perm(dcplus, dcplus, aligner);
dcminus = vec_perm(dcminus, dcminus, aligner);
for (i = 0; i < size; i += 4) {
v0 = vec_ld(0, dst+0*stride);
v1 = vec_ld(0, dst+1*stride);
v2 = vec_ld(0, dst+2*stride);
v3 = vec_ld(0, dst+3*stride);
v0 = vec_adds(v0, dcplus);
v1 = vec_adds(v1, dcplus);
v2 = vec_adds(v2, dcplus);
v3 = vec_adds(v3, dcplus);
v0 = vec_subs(v0, dcminus);
v1 = vec_subs(v1, dcminus);
v2 = vec_subs(v2, dcminus);
v3 = vec_subs(v3, dcminus);
vec_st(v0, 0, dst+0*stride);
vec_st(v1, 0, dst+1*stride);
vec_st(v2, 0, dst+2*stride);
vec_st(v3, 0, dst+3*stride);
dst += 4*stride;
}
}
/* dest*~srca + src */
static force_inline vector unsigned int
over (vector unsigned int src,
vector unsigned int srca,
vector unsigned int dest)
{
vector unsigned char tmp = (vector unsigned char)
pix_multiply (dest, negate (srca));
tmp = vec_adds ((vector unsigned char)src, tmp);
return (vector unsigned int)tmp;
}
static void predict_16x16_p_altivec( uint8_t *src )
{
int H = 0, V = 0;
for( int i = 1; i <= 8; i++ )
{
H += i * ( src[7+i - FDEC_STRIDE ] - src[7-i - FDEC_STRIDE ] );
V += i * ( src[(7+i)*FDEC_STRIDE -1] - src[(7-i)*FDEC_STRIDE -1] );
}
int a = 16 * ( src[15*FDEC_STRIDE -1] + src[15 - FDEC_STRIDE] );
int b = ( 5 * H + 32 ) >> 6;
int c = ( 5 * V + 32 ) >> 6;
int i00 = a - b * 7 - c * 7 + 16;
vec_s16_u i00_u, b_u, c_u;
i00_u.s[0] = i00;
b_u.s[0] = b;
c_u.s[0] = c;
vec_u16_t val5_v = vec_splat_u16(5);
vec_s16_t i00_v, b_v, c_v;
i00_v = vec_splat(i00_u.v, 0);
b_v = vec_splat(b_u.v, 0);
c_v = vec_splat(c_u.v, 0);
vec_s16_t induc_v = (vec_s16_t) CV(0, 1, 2, 3, 4, 5, 6, 7);
vec_s16_t b8_v = vec_sl(b_v, vec_splat_u16(3));
vec_s16_t add_i0_b_0v = vec_mladd(induc_v, b_v, i00_v);
vec_s16_t add_i0_b_8v = vec_adds(b8_v, add_i0_b_0v);
for( int y = 0; y < 16; y++ )
{
vec_s16_t shift_0_v = vec_sra(add_i0_b_0v, val5_v);
vec_s16_t shift_8_v = vec_sra(add_i0_b_8v, val5_v);
vec_u8_t com_sat_v = vec_packsu(shift_0_v, shift_8_v);
vec_st( com_sat_v, 0, &src[0]);
src += FDEC_STRIDE;
add_i0_b_0v = vec_adds(add_i0_b_0v, c_v);
add_i0_b_8v = vec_adds(add_i0_b_8v, c_v);
}
}
static force_inline vector unsigned int
pix_multiply (vector unsigned int p, vector unsigned int a)
{
vector unsigned short hi, lo, mod;
/* unpack to short */
hi = (vector unsigned short)
vec_mergeh ((vector unsigned char)AVV (0),
(vector unsigned char)p);
mod = (vector unsigned short)
vec_mergeh ((vector unsigned char)AVV (0),
(vector unsigned char)a);
hi = vec_mladd (hi, mod, (vector unsigned short)
AVV (0x0080, 0x0080, 0x0080, 0x0080,
0x0080, 0x0080, 0x0080, 0x0080));
hi = vec_adds (hi, vec_sr (hi, vec_splat_u16 (8)));
hi = vec_sr (hi, vec_splat_u16 (8));
/* unpack to short */
lo = (vector unsigned short)
vec_mergel ((vector unsigned char)AVV (0),
(vector unsigned char)p);
mod = (vector unsigned short)
vec_mergel ((vector unsigned char)AVV (0),
(vector unsigned char)a);
lo = vec_mladd (lo, mod, (vector unsigned short)
AVV (0x0080, 0x0080, 0x0080, 0x0080,
0x0080, 0x0080, 0x0080, 0x0080));
lo = vec_adds (lo, vec_sr (lo, vec_splat_u16 (8)));
lo = vec_sr (lo, vec_splat_u16 (8));
return (vector unsigned int)vec_packsu (hi, lo);
}
static void predict_8x8c_p_altivec( uint8_t *src )
{
int H = 0, V = 0;
for( int i = 0; i < 4; i++ )
{
H += ( i + 1 ) * ( src[4+i - FDEC_STRIDE] - src[2 - i -FDEC_STRIDE] );
V += ( i + 1 ) * ( src[-1 +(i+4)*FDEC_STRIDE] - src[-1+(2-i)*FDEC_STRIDE] );
}
int a = 16 * ( src[-1+7*FDEC_STRIDE] + src[7 - FDEC_STRIDE] );
int b = ( 17 * H + 16 ) >> 5;
int c = ( 17 * V + 16 ) >> 5;
int i00 = a -3*b -3*c + 16;
vec_s16_u i00_u, b_u, c_u;
i00_u.s[0] = i00;
b_u.s[0] = b;
c_u.s[0] = c;
vec_u16_t val5_v = vec_splat_u16(5);
vec_s16_t i00_v, b_v, c_v;
i00_v = vec_splat(i00_u.v, 0);
b_v = vec_splat(b_u.v, 0);
c_v = vec_splat(c_u.v, 0);
vec_s16_t induc_v = (vec_s16_t) CV(0, 1, 2, 3, 4, 5, 6, 7);
vec_s16_t add_i0_b_0v = vec_mladd(induc_v, b_v, i00_v);
PREP_STORE8;
for( int i = 0; i < 8; ++i )
{
vec_s16_t shift_0_v = vec_sra(add_i0_b_0v, val5_v);
vec_u8_t com_sat_v = vec_packsu(shift_0_v, shift_0_v);
VEC_STORE8(com_sat_v, &src[0]);
src += FDEC_STRIDE;
add_i0_b_0v = vec_adds(add_i0_b_0v, c_v);
}
}
void imageFilterAddTo_Altivec(unsigned char *dst, unsigned char *src, int length)
{
int n = length;
// Compute first few values so we're on a 16-byte boundary in dst
while( (((long)dst & 0xF) > 0) && (n > 0) ) {
ADDTO_PIXEL();
--n; ++dst; ++src;
}
// Do bulk of processing using Altivec (add 16 8-bit unsigned integers, with saturation)
while(n >= 16) {
vector unsigned char s = vec_ld(0,src);
vector unsigned char d = vec_ld(0,dst);
vector unsigned char r = vec_adds(d, s);
vec_st(r,0,dst);
n -= 16; src += 16; dst += 16;
}
// If any bytes are left over, deal with them individually
++n;
BASIC_ADDTO();
}
void pix_background :: processYUVAltivec(imageStruct &image)
{
register int h,w,i,j,width;
int pixsize = image.xsize * image.ysize * image.csize;
h = image.ysize;
w = image.xsize/8;
width = image.xsize/8;
//check to see if the buffer isn't 16byte aligned (highly unlikely)
if (image.ysize*image.xsize % 16 != 0){
error("image not properly aligned for Altivec - try something SD or HD maybe?");
return;
}
union{
unsigned short s[8];
vector unsigned short v;
}shortBuffer;
if(m_savedImage.xsize!=image.xsize ||
m_savedImage.ysize!=image.ysize ||
m_savedImage.format!=image.format)m_reset=1;
m_savedImage.xsize=image.xsize;
m_savedImage.ysize=image.ysize;
m_savedImage.setCsizeByFormat(image.format);
m_savedImage.reallocate();
if (m_reset){
memcpy(m_savedImage.data,image.data,pixsize);
m_reset = 0;
}
register vector unsigned short UVres1, Yres1, UVres2, Yres2;//interleave;
register vector unsigned short hiImage, loImage;
register vector unsigned short Yrange, UVrange, Yblank,UVblank,blank;
register vector bool short Ymasklo,Ymaskhi, UVmaskhi;
register vector unsigned short Yhi,Ylo,UVhi,UVlo;
register vector unsigned char one = vec_splat_u8(1);
register vector unsigned short sone = vec_splat_u16(1);
register vector unsigned int Uhi, Ulo, Vhi, Vlo,Ures,Vres;
register vector bool int Umasklo, Umaskhi, Vmaskhi, Vmasklo;
vector unsigned char *inData = (vector unsigned char*) image.data;
vector unsigned char *rightData = (vector unsigned char*) m_savedImage.data;
shortBuffer.s[0] = m_Yrange;
Yrange = shortBuffer.v;
Yrange = vec_splat(Yrange,0);
shortBuffer.s[0] = 128;
shortBuffer.s[1] = 0;
shortBuffer.s[2] = 128;
shortBuffer.s[3] = 0;
shortBuffer.s[4] = 128;
shortBuffer.s[5] = 0;
shortBuffer.s[6] = 128;
shortBuffer.s[7] = 0;
blank = shortBuffer.v;
shortBuffer.s[0] = 0;
Yblank = shortBuffer.v;
Yblank = vec_splat(Yblank,0);
shortBuffer.s[0] = 128;
UVblank = shortBuffer.v;
UVblank = vec_splat(UVblank,0);
shortBuffer.s[0] = m_Urange;
shortBuffer.s[1] = m_Vrange;
shortBuffer.s[2] = m_Urange;
shortBuffer.s[3] = m_Vrange;
shortBuffer.s[4] = m_Urange;
shortBuffer.s[5] = m_Vrange;
shortBuffer.s[6] = m_Urange;
shortBuffer.s[7] = m_Vrange;
UVrange = shortBuffer.v;
//setup the cache prefetch -- A MUST!!!
UInt32 prefetchSize = GetPrefetchConstant( 16, 1, 256 );
#ifndef PPC970
vec_dst( inData, prefetchSize, 0 );
vec_dst( rightData, prefetchSize, 1 );
vec_dst( inData+32, prefetchSize, 2 );
vec_dst( rightData+32, prefetchSize, 3 );
#endif //PPC970
for ( i=0; i<h; i++){
for (j=0; j<w; j++)
{
#ifndef PPC970
//this function is probably memory bound on most G4's -- what else is new?
vec_dst( inData, prefetchSize, 0 );
vec_dst( rightData, prefetchSize, 1 );
vec_dst( inData+32, prefetchSize, 2 );
vec_dst( rightData+32, prefetchSize, 3 );
#endif
//separate the U and V from Y
UVres1 = (vector unsigned short)vec_mule(one,inData[0]);
UVres2 = (vector unsigned short)vec_mule(one,rightData[0]);
//vec_mulo Y * 1 to short vector Y Y Y Y shorts
Yres1 = (vector unsigned short)vec_mulo(one,inData[0]);
Yres2 = (vector unsigned short)vec_mulo(one,rightData[0]);
Yhi = vec_adds(Yres2,Yrange);
Ylo = vec_subs(Yres2,Yrange);
//go to ints for comparison
UVhi = vec_adds(UVres2,UVrange);
UVlo = vec_subs(UVres2,UVrange);
Uhi = vec_mule(sone,UVhi);
Ulo = vec_mule(sone,UVlo);
Vhi = vec_mulo(sone,UVhi);
Vlo = vec_mulo(sone,UVlo);
Ures = vec_mule(sone,UVres1);
Vres = vec_mulo(sone,UVres1);
Umasklo = vec_cmpgt(Ures,Ulo);
Umaskhi = vec_cmplt(Ures,Uhi);
Vmasklo = vec_cmpgt(Vres,Vlo);
Vmaskhi = vec_cmplt(Vres,Vhi);
Umaskhi = vec_and(Umaskhi,Umasklo);
Vmaskhi = vec_and(Vmaskhi,Vmasklo);
Umasklo = vec_and(Umaskhi,Vmaskhi);
Vmasklo = vec_and(Umaskhi,Vmaskhi);
hiImage = (vector unsigned short)vec_mergeh(Umasklo,Vmasklo);
loImage = (vector unsigned short)vec_mergel(Umasklo,Vmasklo);
//pack it back down to bool short
UVmaskhi = (vector bool short)vec_packsu(hiImage,loImage);
Ymasklo = vec_cmpgt(Yres1,Ylo);
Ymaskhi = vec_cmplt(Yres1,Yhi);
Ymaskhi = vec_and(Ymaskhi,Ymasklo);
Ymaskhi = vec_and(Ymaskhi,UVmaskhi);
UVmaskhi = vec_and(Ymaskhi,UVmaskhi);
//bitwise comparison and move using the result of the comparison as a mask
Yres1 = vec_sel(Yres1,Yblank,Ymaskhi);
//UVres1 = vec_sel(UVres1,UVres2,UVmaskhi);
UVres1 = vec_sel(UVres1,UVblank,UVmaskhi);
//merge the Y and UV back together
hiImage = vec_mergeh(UVres1,Yres1);
loImage = vec_mergel(UVres1,Yres1);
//pack it back down to unsigned char to store
inData[0] = vec_packsu(hiImage,loImage);
inData++;
rightData++;
}
#ifndef PPC970
vec_dss(0);
vec_dss(1);
vec_dss(2);
vec_dss(3);
#endif
}
}
void
dct_vmx (vector signed short *input, vector signed short *output,
vector signed short *postscale)
{
vector signed short mul0, mul1, mul2, mul3, mul4, mul5, mul6, mul;
vector signed short v0, v1, v2, v3, v4, v5, v6, v7, v8, v9;
vector signed short v20, v21, v22, v23, v24, v25, v26, v27, v31;
int i;
vector signed short in[8], out[8];
/* Load first eight rows of input data */
/* Load multiplication constants */
/* Splat multiplication constants */
mul0 = vec_splat(input[8],0);
mul1 = vec_splat(input[8],1);
mul2 = vec_splat(input[8],2);
mul3 = vec_splat(input[8],3);
mul4 = vec_splat(input[8],4);
mul5 = vec_splat(input[8],5);
mul6 = vec_splat(input[8],6);
/* Perform DCT on the eight columns */
/*********** Stage 1 ***********/
v8 = vec_adds (input[0], input[7]);
v9 = vec_subs (input[0], input[7]);
v0 = vec_adds (input[1], input[6]);
v7 = vec_subs (input[1], input[6]);
v1 = vec_adds (input[2], input[5]);
v6 = vec_subs (input[2], input[5]);
v2 = vec_adds (input[3], input[4]);
v5 = vec_subs (input[3], input[4]);
/*********** Stage 2 ***********/
/* Top */
v3 = vec_adds (v8, v2); /* (V0+V7) + (V3+V4) */
v4 = vec_subs (v8, v2); /* (V0+V7) - (V3+V4) */
v2 = vec_adds (v0, v1); /* (V1+V6) + (V2+V5) */
v8 = vec_subs (v0, v1); /* (V1+V6) - (V2+V5) */
/* Bottom */
v0 = vec_subs (v7, v6); /* (V1-V6) - (V2-V5) */
v1 = vec_adds (v7, v6); /* (V1-V6) + (V2-V5) */
/*********** Stage 3 ***********/
/* Top */
in[0] = vec_adds (v3, v2); /* y0 = v3 + v2 */
in[4] = vec_subs (v3, v2); /* y4 = v3 - v2 */
in[2] = vec_mradds (v8, mul2, v4); /* y2 = v8 * a0 + v4 */
v6 = vec_mradds (v4, mul2, mul6);
in[6] = vec_subs (v6, v8); /* y6 = v4 * a0 - v8 */
/* Bottom */
v6 = vec_mradds (v0, mul0, v5); /* v6 = v0 * (c4) + v5 */
v7 = vec_mradds (v0, mul4, v5); /* v7 = v0 * (-c4) + v5 */
v2 = vec_mradds (v1, mul4, v9); /* v2 = v1 * (-c4) + v9 */
v3 = vec_mradds (v1, mul0, v9); /* v3 = v1 * (c4) + v9 */
/*********** Stage 4 ***********/
/* Bottom */
in[1] = vec_mradds (v6, mul3, v3); /* y1 = v6 * (a1) + v3 */
v23 = vec_mradds (v3, mul3, mul6);
in[7] = vec_subs (v23, v6); /* y7 = v3 * (a1) - v6 */
in[5] = vec_mradds (v2, mul1, v7); /* y5 = v2 * (a2) + v7 */
in[3] = vec_mradds (v7, mul5, v2); /* y3 = v7 * (-a2) + v2 */
transpose_vmx (in, out);
/* Perform DCT on the eight rows */
/*********** Stage 1 ***********/
v8 = vec_adds (out[0], out[7]);
v9 = vec_subs (out[0], out[7]);
v0 = vec_adds (out[1], out[6]);
v7 = vec_subs (out[1], out[6]);
v1 = vec_adds (out[2], out[5]);
v6 = vec_subs (out[2], out[5]);
v2 = vec_adds (out[3], out[4]);
v5 = vec_subs (out[3], out[4]);
/*********** Stage 2 ***********/
/* Top */
v3 = vec_adds (v8, v2); /* (V0+V7) + (V3+V4) */
v4 = vec_subs (v8, v2); /* (V0+V7) - (V3+V4) */
v2 = vec_adds (v0, v1); /* (V1+V6) + (V2+V5) */
v8 = vec_subs (v0, v1); /* (V1+V6) - (V2+V5) */
/* Bottom */
v0 = vec_subs (v7, v6); /* (V1-V6) - (V2-V5) */
v1 = vec_adds (v7, v6); /* (V1-V6) + (V2-V5) */
/*********** Stage 3 ***********/
/* Top */
v25 = vec_subs (v25, v25); /* reinit v25 = 0 */
v20 = vec_adds (v3, v2); /* y0 = v3 + v2 */
v24 = vec_subs (v3, v2); /* y4 = v3 - v2 */
v22 = vec_mradds (v8, mul2, v4); /* y2 = v8 * a0 + v4 */
v6 = vec_mradds (v4, mul2, v25);
v26 = vec_subs (v6, v8); /* y6 = v4 * a0 - v8 */
/* Bottom */
v6 = vec_mradds (v0, mul0, v5); /* v6 = v0 * (c4) + v5 */
v7 = vec_mradds (v0, mul4, v5); /* v7 = v0 * (-c4) + v5 */
v2 = vec_mradds (v1, mul4, v9); /* v2 = v1 * (-c4) + v9 */
v3 = vec_mradds (v1, mul0, v9); /* v3 = v1 * (c4) + v9 */
/*********** Stage 4 ***********/
/* Bottom */
v21 = vec_mradds (v6, mul3, v3); /* y1 = v6 * (a1) + v3 */
v23 = vec_mradds (v3, mul3, v25);
v27 = vec_subs (v23, v6); /* y7 = v3 * (a1) - v6 */
v25 = vec_mradds (v2, mul1, v7); /* y5 = v2 * (a2) + v7 */
v23 = vec_mradds (v7, mul5, v2); /* y3 = v7 * (-a2) + v2 */
/* Post-scale and store reults */
v31 = vec_subs (v31, v31); /* reinit v25 = 0 */
output[0] = vec_mradds (postscale[0], v20, v31);
output[2] = vec_mradds (postscale[2], v22, v31);
output[4] = vec_mradds (postscale[4], v24, v31);
output[6] = vec_mradds (postscale[6], v26, v31);
output[1] = vec_mradds (postscale[1], v21, v31);
output[3] = vec_mradds (postscale[3], v23, v31);
output[5] = vec_mradds (postscale[5], v25, v31);
output[7] = vec_mradds (postscale[7], v27, v31);
}
void pix_add :: processYUV_Altivec(imageStruct &image, imageStruct &right)
{
int h,w,width;
width = image.xsize/8;
//format is U Y V Y
union
{
//unsigned int i;
short elements[8];
//vector signed char v;
vector signed short v;
}shortBuffer;
union
{
//unsigned int i;
unsigned char elements[16];
//vector signed char v;
vector unsigned char v;
}charBuffer;
//vector unsigned char c;
register vector signed short d, hiImage, loImage, YRight, UVRight, YImage, UVImage, UVTemp, YTemp;
// vector unsigned char zero = vec_splat_u8(0);
register vector unsigned char c,one;
// vector signed short zshort = vec_splat_s16(0);
vector unsigned char *inData = (vector unsigned char*) image.data;
vector unsigned char *rightData = (vector unsigned char*) right.data;
//Write the pixel (pair) to the transfer buffer
charBuffer.elements[0] = 2;
charBuffer.elements[1] = 1;
charBuffer.elements[2] = 2;
charBuffer.elements[3] = 1;
charBuffer.elements[4] = 2;
charBuffer.elements[5] = 1;
charBuffer.elements[6] = 2;
charBuffer.elements[7] = 1;
charBuffer.elements[8] = 2;
charBuffer.elements[9] = 1;
charBuffer.elements[10] = 2;
charBuffer.elements[11] = 1;
charBuffer.elements[12] = 2;
charBuffer.elements[13] = 1;
charBuffer.elements[14] = 2;
charBuffer.elements[15] = 1;
//Load it into the vector unit
c = charBuffer.v;
one = vec_splat_u8( 1 );
shortBuffer.elements[0] = 255;
//Load it into the vector unit
d = shortBuffer.v;
d = static_cast<vector signed short>(vec_splat(static_cast<vector signed short>(d),0));
#ifndef PPC970
UInt32 prefetchSize = GetPrefetchConstant( 16, 1, 256 );
vec_dst( inData, prefetchSize, 0 );
vec_dst( rightData, prefetchSize, 1 );
#endif
for ( h=0; h<image.ysize; h++){
for (w=0; w<width; w++)
{
#ifndef PPC970
vec_dst( inData, prefetchSize, 0 );
vec_dst( rightData, prefetchSize, 1 );
#endif
//interleaved U Y V Y chars
//vec_mule UV * 2 to short vector U V U V shorts
UVImage = static_cast<vector signed short>(vec_mule(one,inData[0]));
UVRight = static_cast<vector signed short>(vec_mule(c,rightData[0]));
//vec_mulo Y * 1 to short vector Y Y Y Y shorts
YImage = static_cast<vector signed short>(vec_mulo(c,inData[0]));
YRight = static_cast<vector signed short>(vec_mulo(c,rightData[0]));
//vel_subs UV - 255
UVRight = static_cast<vector signed short>(vec_subs(UVRight, d));
//vec_adds UV
UVTemp = vec_adds(UVImage,UVRight);
//vec_adds Y
YTemp = vec_adds(YImage,YRight);
hiImage = vec_mergeh(UVTemp,YTemp);
loImage = vec_mergel(UVTemp,YTemp);
//vec_mergel + vec_mergeh Y and UV
inData[0] = vec_packsu(hiImage, loImage);
inData++;
rightData++;
}
#ifndef PPC970
vec_dss( 0 );
vec_dss( 1 );
#endif
} /*end of working altivec function */
}
/* Function: p7_ViterbiFilter()
* Synopsis: Calculates Viterbi score, vewy vewy fast, in limited precision.
* Incept: SRE, Tue Nov 27 09:15:24 2007 [Janelia]
*
* Purpose: Calculates an approximation of the Viterbi score for sequence
* <dsq> of length <L> residues, using optimized profile <om>,
* and a preallocated one-row DP matrix <ox>. Return the
* estimated Viterbi score (in nats) in <ret_sc>.
*
* Score may overflow (and will, on high-scoring
* sequences), but will not underflow.
*
* The model must be in a local alignment mode; other modes
* cannot provide the necessary guarantee of no underflow.
*
* This is a striped SIMD Viterbi implementation using Intel
* VMX integer intrinsics \citep{Farrar07}, in reduced
* precision (signed words, 16 bits).
*
* Args: dsq - digital target sequence, 1..L
* L - length of dsq in residues
* om - optimized profile
* ox - DP matrix
* ret_sc - RETURN: Viterbi score (in nats)
*
* Returns: <eslOK> on success;
* <eslERANGE> if the score overflows; in this case
* <*ret_sc> is <eslINFINITY>, and the sequence can
* be treated as a high-scoring hit.
*
* Throws: <eslEINVAL> if <ox> allocation is too small, or if
* profile isn't in a local alignment mode. (Must be in local
* alignment mode because that's what helps us guarantee
* limited dynamic range.)
*
* Xref: [Farrar07] for ideas behind striped SIMD DP.
* J2/46-47 for layout of HMMER's striped SIMD DP.
* J2/50 for single row DP.
* J2/60 for reduced precision (epu8)
* J2/65 for initial benchmarking
* J2/66 for precision maximization
* J4/138-140 for reimplementation in 16-bit precision
*/
int
p7_ViterbiFilter(const ESL_DSQ *dsq, int L, const P7_OPROFILE *om, P7_OMX *ox, float *ret_sc)
{
vector signed short mpv, dpv, ipv; /* previous row values */
vector signed short sv; /* temp storage of 1 curr row value in progress */
vector signed short dcv; /* delayed storage of D(i,q+1) */
vector signed short xEv; /* E state: keeps max for Mk->E as we go */
vector signed short xBv; /* B state: splatted vector of B[i-1] for B->Mk calculations */
vector signed short Dmaxv; /* keeps track of maximum D cell on row */
int16_t xE, xB, xC, xJ, xN; /* special states' scores */
int16_t Dmax; /* maximum D cell score on row */
int i; /* counter over sequence positions 1..L */
int q; /* counter over vectors 0..nq-1 */
int Q; /* segment length: # of vectors */
vector signed short *dp; /* using {MDI}MX(q) macro requires initialization of <dp> */
vector signed short *rsc; /* will point at om->ru[x] for residue x[i] */
vector signed short *tsc; /* will point into (and step thru) om->tu */
vector signed short negInfv;
Q = p7O_NQW(om->M);
dp = ox->dpw[0];
/* Check that the DP matrix is ok for us. */
if (Q > ox->allocQ8) ESL_EXCEPTION(eslEINVAL, "DP matrix allocated too small");
if (om->mode != p7_LOCAL && om->mode != p7_UNILOCAL) ESL_EXCEPTION(eslEINVAL, "Fast filter only works for local alignment");
ox->M = om->M;
negInfv = esl_vmx_set_s16((signed short)-32768);
/* Initialization. In unsigned arithmetic, -infinity is -32768
*/
for (q = 0; q < Q; q++)
MMXo(q) = IMXo(q) = DMXo(q) = negInfv;
xN = om->base_w;
xB = xN + om->xw[p7O_N][p7O_MOVE];
xJ = -32768;
xC = -32768;
xE = -32768;
#if p7_DEBUGGING
if (ox->debugging) p7_omx_DumpVFRow(ox, 0, xE, 0, xJ, xB, xC); /* first 0 is <rowi>: do header. second 0 is xN: always 0 here. */
#endif
for (i = 1; i <= L; i++)
{
rsc = om->rwv[dsq[i]];
tsc = om->twv;
dcv = negInfv; /* "-infinity" */
xEv = negInfv;
Dmaxv = negInfv;
xBv = esl_vmx_set_s16(xB);
/* Right shifts by 1 value (2 bytes). 4,8,12,x becomes x,4,8,12.
* Because ia32 is littlendian, this means a left bit shift.
* Zeros shift on automatically; replace it with -32768.
*/
mpv = MMXo(Q-1); mpv = vec_sld(negInfv, mpv, 14);
dpv = DMXo(Q-1); dpv = vec_sld(negInfv, dpv, 14);
ipv = IMXo(Q-1); ipv = vec_sld(negInfv, ipv, 14);
for (q = 0; q < Q; q++)
{
/* Calculate new MMXo(i,q); don't store it yet, hold it in sv. */
sv = vec_adds(xBv, *tsc); tsc++;
sv = vec_max (sv, vec_adds(mpv, *tsc)); tsc++;
sv = vec_max (sv, vec_adds(ipv, *tsc)); tsc++;
sv = vec_max (sv, vec_adds(dpv, *tsc)); tsc++;
sv = vec_adds(sv, *rsc); rsc++;
xEv = vec_max(xEv, sv);
/* Load {MDI}(i-1,q) into mpv, dpv, ipv;
* {MDI}MX(q) is then the current, not the prev row
*/
mpv = MMXo(q);
dpv = DMXo(q);
ipv = IMXo(q);
/* Do the delayed stores of {MD}(i,q) now that memory is usable */
MMXo(q) = sv;
DMXo(q) = dcv;
/* Calculate the next D(i,q+1) partially: M->D only;
* delay storage, holding it in dcv
*/
dcv = vec_adds(sv, *tsc); tsc++;
Dmaxv = vec_max(dcv, Dmaxv);
/* Calculate and store I(i,q) */
sv = vec_adds(mpv, *tsc); tsc++;
IMXo(q)= vec_max(sv, vec_adds(ipv, *tsc)); tsc++;
}
/* Now the "special" states, which start from Mk->E (->C, ->J->B) */
xE = esl_vmx_hmax_s16(xEv);
if (xE >= 32767) { *ret_sc = eslINFINITY; return eslERANGE; } /* immediately detect overflow */
xN = xN + om->xw[p7O_N][p7O_LOOP];
xC = ESL_MAX(xC + om->xw[p7O_C][p7O_LOOP], xE + om->xw[p7O_E][p7O_MOVE]);
xJ = ESL_MAX(xJ + om->xw[p7O_J][p7O_LOOP], xE + om->xw[p7O_E][p7O_LOOP]);
xB = ESL_MAX(xJ + om->xw[p7O_J][p7O_MOVE], xN + om->xw[p7O_N][p7O_MOVE]);
/* and now xB will carry over into next i, and xC carries over after i=L */
/* Finally the "lazy F" loop (sensu [Farrar07]). We can often
* prove that we don't need to evaluate any D->D paths at all.
*
* The observation is that if we can show that on the next row,
* B->M(i+1,k) paths always dominate M->D->...->D->M(i+1,k) paths
* for all k, then we don't need any D->D calculations.
*
* The test condition is:
* max_k D(i,k) + max_k ( TDD(k-2) + TDM(k-1) - TBM(k) ) < xB(i)
* So:
* max_k (TDD(k-2) + TDM(k-1) - TBM(k)) is precalc'ed in om->dd_bound;
* max_k D(i,k) is why we tracked Dmaxv;
* xB(i) was just calculated above.
*/
Dmax = esl_vmx_hmax_s16(Dmaxv);
if (Dmax + om->ddbound_w > xB)
{
/* Now we're obligated to do at least one complete DD path to be sure. */
/* dcv has carried through from end of q loop above */
dcv = vec_sld(negInfv, dcv, 14);
tsc = om->twv + 7*Q; /* set tsc to start of the DD's */
for (q = 0; q < Q; q++)
{
DMXo(q) = vec_max(dcv, DMXo(q));
dcv = vec_adds(DMXo(q), *tsc); tsc++;
}
/* We may have to do up to three more passes; the check
* is for whether crossing a segment boundary can improve
* our score.
*/
do {
dcv = vec_sld(negInfv, dcv, 14);
tsc = om->twv + 7*Q; /* set tsc to start of the DD's */
for (q = 0; q < Q; q++)
{
if (! vec_any_gt(dcv, DMXo(q))) break;
DMXo(q) = vec_max(dcv, DMXo(q));
dcv = vec_adds(DMXo(q), *tsc); tsc++;
}
} while (q == Q);
}
else /* not calculating DD? then just store the last M->D vector calc'ed.*/
DMXo(0) = vec_sld(negInfv, dcv, 14);
#if p7_DEBUGGING
if (ox->debugging) p7_omx_DumpVFRow(ox, i, xE, 0, xJ, xB, xC);
#endif
} /* end loop over sequence residues 1..L */
/* finally C->T */
if (xC > -32768)
{
*ret_sc = (float) xC + (float) om->xw[p7O_C][p7O_MOVE] - (float) om->base_w;
/* *ret_sc += L * om->ncj_roundoff; see J4/150 for rationale: superceded by -3.0nat approximation*/
*ret_sc /= om->scale_w;
*ret_sc -= 3.0; /* the NN/CC/JJ=0,-3nat approximation: see J5/36. That's ~ L \log \frac{L}{L+3}, for our NN,CC,JJ contrib */
}
else *ret_sc = -eslINFINITY;
return eslOK;
}
static void PREFIX_h264_qpel16_hv_lowpass_altivec(uint8_t * dst, int16_t * tmp, uint8_t * src, int dstStride, int tmpStride, int srcStride) {
register int i;
LOAD_ZERO;
const vec_u8 permM2 = vec_lvsl(-2, src);
const vec_u8 permM1 = vec_lvsl(-1, src);
const vec_u8 permP0 = vec_lvsl(+0, src);
const vec_u8 permP1 = vec_lvsl(+1, src);
const vec_u8 permP2 = vec_lvsl(+2, src);
const vec_u8 permP3 = vec_lvsl(+3, src);
const vec_s16 v20ss = vec_sl(vec_splat_s16(5),vec_splat_u16(2));
const vec_u32 v10ui = vec_splat_u32(10);
const vec_s16 v5ss = vec_splat_s16(5);
const vec_s16 v1ss = vec_splat_s16(1);
const vec_s32 v512si = vec_sl(vec_splat_s32(1),vec_splat_u32(9));
const vec_u32 v16ui = vec_sl(vec_splat_u32(1),vec_splat_u32(4));
register int align = ((((unsigned long)src) - 2) % 16);
vec_s16 srcP0A, srcP0B, srcP1A, srcP1B,
srcP2A, srcP2B, srcP3A, srcP3B,
srcM1A, srcM1B, srcM2A, srcM2B,
sum1A, sum1B, sum2A, sum2B, sum3A, sum3B,
pp1A, pp1B, pp2A, pp2B, psumA, psumB;
const vec_u8 mperm = (const vec_u8)
{0x00, 0x08, 0x01, 0x09, 0x02, 0x0A, 0x03, 0x0B,
0x04, 0x0C, 0x05, 0x0D, 0x06, 0x0E, 0x07, 0x0F};
int16_t *tmpbis = tmp;
vec_s16 tmpM1ssA, tmpM1ssB, tmpM2ssA, tmpM2ssB,
tmpP0ssA, tmpP0ssB, tmpP1ssA, tmpP1ssB,
tmpP2ssA, tmpP2ssB;
vec_s32 pp1Ae, pp1Ao, pp1Be, pp1Bo, pp2Ae, pp2Ao, pp2Be, pp2Bo,
pp3Ae, pp3Ao, pp3Be, pp3Bo, pp1cAe, pp1cAo, pp1cBe, pp1cBo,
pp32Ae, pp32Ao, pp32Be, pp32Bo, sumAe, sumAo, sumBe, sumBo,
ssumAe, ssumAo, ssumBe, ssumBo;
vec_u8 fsum, sumv, sum;
vec_s16 ssume, ssumo;
src -= (2 * srcStride);
for (i = 0 ; i < 21 ; i ++) {
vec_u8 srcM2, srcM1, srcP0, srcP1, srcP2, srcP3;
vec_u8 srcR1 = vec_ld(-2, src);
vec_u8 srcR2 = vec_ld(14, src);
switch (align) {
default: {
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = vec_perm(srcR1, srcR2, permP2);
srcP3 = vec_perm(srcR1, srcR2, permP3);
} break;
case 11: {
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = vec_perm(srcR1, srcR2, permP2);
srcP3 = srcR2;
} break;
case 12: {
vec_u8 srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = srcR2;
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 13: {
vec_u8 srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = srcR2;
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 14: {
vec_u8 srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = srcR2;
srcP1 = vec_perm(srcR2, srcR3, permP1);
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 15: {
vec_u8 srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = srcR2;
srcP0 = vec_perm(srcR2, srcR3, permP0);
srcP1 = vec_perm(srcR2, srcR3, permP1);
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
}
srcP0A = (vec_s16) vec_mergeh(zero_u8v, srcP0);
srcP0B = (vec_s16) vec_mergel(zero_u8v, srcP0);
srcP1A = (vec_s16) vec_mergeh(zero_u8v, srcP1);
srcP1B = (vec_s16) vec_mergel(zero_u8v, srcP1);
srcP2A = (vec_s16) vec_mergeh(zero_u8v, srcP2);
srcP2B = (vec_s16) vec_mergel(zero_u8v, srcP2);
srcP3A = (vec_s16) vec_mergeh(zero_u8v, srcP3);
srcP3B = (vec_s16) vec_mergel(zero_u8v, srcP3);
srcM1A = (vec_s16) vec_mergeh(zero_u8v, srcM1);
srcM1B = (vec_s16) vec_mergel(zero_u8v, srcM1);
srcM2A = (vec_s16) vec_mergeh(zero_u8v, srcM2);
srcM2B = (vec_s16) vec_mergel(zero_u8v, srcM2);
sum1A = vec_adds(srcP0A, srcP1A);
sum1B = vec_adds(srcP0B, srcP1B);
sum2A = vec_adds(srcM1A, srcP2A);
sum2B = vec_adds(srcM1B, srcP2B);
sum3A = vec_adds(srcM2A, srcP3A);
sum3B = vec_adds(srcM2B, srcP3B);
pp1A = vec_mladd(sum1A, v20ss, sum3A);
pp1B = vec_mladd(sum1B, v20ss, sum3B);
pp2A = vec_mladd(sum2A, v5ss, zero_s16v);
pp2B = vec_mladd(sum2B, v5ss, zero_s16v);
psumA = vec_sub(pp1A, pp2A);
psumB = vec_sub(pp1B, pp2B);
vec_st(psumA, 0, tmp);
vec_st(psumB, 16, tmp);
src += srcStride;
tmp += tmpStride; /* int16_t*, and stride is 16, so it's OK here */
}
tmpM2ssA = vec_ld(0, tmpbis);
tmpM2ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
tmpM1ssA = vec_ld(0, tmpbis);
tmpM1ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
tmpP0ssA = vec_ld(0, tmpbis);
tmpP0ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
tmpP1ssA = vec_ld(0, tmpbis);
tmpP1ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
tmpP2ssA = vec_ld(0, tmpbis);
tmpP2ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
for (i = 0 ; i < 16 ; i++) {
const vec_s16 tmpP3ssA = vec_ld(0, tmpbis);
const vec_s16 tmpP3ssB = vec_ld(16, tmpbis);
const vec_s16 sum1A = vec_adds(tmpP0ssA, tmpP1ssA);
const vec_s16 sum1B = vec_adds(tmpP0ssB, tmpP1ssB);
const vec_s16 sum2A = vec_adds(tmpM1ssA, tmpP2ssA);
const vec_s16 sum2B = vec_adds(tmpM1ssB, tmpP2ssB);
const vec_s16 sum3A = vec_adds(tmpM2ssA, tmpP3ssA);
const vec_s16 sum3B = vec_adds(tmpM2ssB, tmpP3ssB);
tmpbis += tmpStride;
tmpM2ssA = tmpM1ssA;
tmpM2ssB = tmpM1ssB;
tmpM1ssA = tmpP0ssA;
tmpM1ssB = tmpP0ssB;
tmpP0ssA = tmpP1ssA;
tmpP0ssB = tmpP1ssB;
tmpP1ssA = tmpP2ssA;
tmpP1ssB = tmpP2ssB;
tmpP2ssA = tmpP3ssA;
tmpP2ssB = tmpP3ssB;
pp1Ae = vec_mule(sum1A, v20ss);
pp1Ao = vec_mulo(sum1A, v20ss);
pp1Be = vec_mule(sum1B, v20ss);
pp1Bo = vec_mulo(sum1B, v20ss);
pp2Ae = vec_mule(sum2A, v5ss);
pp2Ao = vec_mulo(sum2A, v5ss);
pp2Be = vec_mule(sum2B, v5ss);
pp2Bo = vec_mulo(sum2B, v5ss);
pp3Ae = vec_sra((vec_s32)sum3A, v16ui);
pp3Ao = vec_mulo(sum3A, v1ss);
pp3Be = vec_sra((vec_s32)sum3B, v16ui);
pp3Bo = vec_mulo(sum3B, v1ss);
pp1cAe = vec_add(pp1Ae, v512si);
pp1cAo = vec_add(pp1Ao, v512si);
pp1cBe = vec_add(pp1Be, v512si);
pp1cBo = vec_add(pp1Bo, v512si);
pp32Ae = vec_sub(pp3Ae, pp2Ae);
pp32Ao = vec_sub(pp3Ao, pp2Ao);
pp32Be = vec_sub(pp3Be, pp2Be);
pp32Bo = vec_sub(pp3Bo, pp2Bo);
sumAe = vec_add(pp1cAe, pp32Ae);
sumAo = vec_add(pp1cAo, pp32Ao);
sumBe = vec_add(pp1cBe, pp32Be);
sumBo = vec_add(pp1cBo, pp32Bo);
ssumAe = vec_sra(sumAe, v10ui);
ssumAo = vec_sra(sumAo, v10ui);
ssumBe = vec_sra(sumBe, v10ui);
ssumBo = vec_sra(sumBo, v10ui);
ssume = vec_packs(ssumAe, ssumBe);
ssumo = vec_packs(ssumAo, ssumBo);
sumv = vec_packsu(ssume, ssumo);
sum = vec_perm(sumv, sumv, mperm);
ASSERT_ALIGNED(dst);
OP_U8_ALTIVEC(fsum, sum, vec_ld(0, dst));
vec_st(fsum, 0, dst);
dst += dstStride;
}
}
/* this code assume stride % 16 == 0 */
static void PREFIX_h264_qpel16_h_lowpass_altivec(uint8_t * dst, uint8_t * src, int dstStride, int srcStride) {
POWERPC_PERF_DECLARE(PREFIX_h264_qpel16_h_lowpass_num, 1);
POWERPC_PERF_START_COUNT(PREFIX_h264_qpel16_h_lowpass_num, 1);
register int i;
const vector signed int vzero = vec_splat_s32(0);
const vector unsigned char permM2 = vec_lvsl(-2, src);
const vector unsigned char permM1 = vec_lvsl(-1, src);
const vector unsigned char permP0 = vec_lvsl(+0, src);
const vector unsigned char permP1 = vec_lvsl(+1, src);
const vector unsigned char permP2 = vec_lvsl(+2, src);
const vector unsigned char permP3 = vec_lvsl(+3, src);
const vector signed short v20ss = (const vector signed short)AVV(20);
const vector unsigned short v5us = vec_splat_u16(5);
const vector signed short v5ss = vec_splat_s16(5);
const vector signed short v16ss = (const vector signed short)AVV(16);
const vector unsigned char dstperm = vec_lvsr(0, dst);
const vector unsigned char neg1 = (const vector unsigned char)vec_splat_s8(-1);
const vector unsigned char dstmask = vec_perm((const vector unsigned char)vzero, neg1, dstperm);
register int align = ((((unsigned long)src) - 2) % 16);
for (i = 0 ; i < 16 ; i ++) {
vector unsigned char srcM2, srcM1, srcP0, srcP1, srcP2, srcP3;
vector unsigned char srcR1 = vec_ld(-2, src);
vector unsigned char srcR2 = vec_ld(14, src);
switch (align) {
default: {
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = vec_perm(srcR1, srcR2, permP2);
srcP3 = vec_perm(srcR1, srcR2, permP3);
} break;
case 11: {
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = vec_perm(srcR1, srcR2, permP2);
srcP3 = srcR2;
} break;
case 12: {
vector unsigned char srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = srcR2;
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 13: {
vector unsigned char srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = srcR2;
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 14: {
vector unsigned char srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = srcR2;
srcP1 = vec_perm(srcR2, srcR3, permP1);
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 15: {
vector unsigned char srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = srcR2;
srcP0 = vec_perm(srcR2, srcR3, permP0);
srcP1 = vec_perm(srcR2, srcR3, permP1);
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
}
const vector signed short srcP0A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP0);
const vector signed short srcP0B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP0);
const vector signed short srcP1A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP1);
const vector signed short srcP1B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP1);
const vector signed short srcP2A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP2);
const vector signed short srcP2B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP2);
const vector signed short srcP3A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP3);
const vector signed short srcP3B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP3);
const vector signed short srcM1A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcM1);
const vector signed short srcM1B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcM1);
const vector signed short srcM2A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcM2);
const vector signed short srcM2B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcM2);
const vector signed short sum1A = vec_adds(srcP0A, srcP1A);
const vector signed short sum1B = vec_adds(srcP0B, srcP1B);
const vector signed short sum2A = vec_adds(srcM1A, srcP2A);
const vector signed short sum2B = vec_adds(srcM1B, srcP2B);
const vector signed short sum3A = vec_adds(srcM2A, srcP3A);
const vector signed short sum3B = vec_adds(srcM2B, srcP3B);
const vector signed short pp1A = vec_mladd(sum1A, v20ss, v16ss);
const vector signed short pp1B = vec_mladd(sum1B, v20ss, v16ss);
const vector signed short pp2A = vec_mladd(sum2A, v5ss, (vector signed short)vzero);
const vector signed short pp2B = vec_mladd(sum2B, v5ss, (vector signed short)vzero);
const vector signed short pp3A = vec_add(sum3A, pp1A);
const vector signed short pp3B = vec_add(sum3B, pp1B);
const vector signed short psumA = vec_sub(pp3A, pp2A);
const vector signed short psumB = vec_sub(pp3B, pp2B);
const vector signed short sumA = vec_sra(psumA, v5us);
const vector signed short sumB = vec_sra(psumB, v5us);
const vector unsigned char sum = vec_packsu(sumA, sumB);
const vector unsigned char dst1 = vec_ld(0, dst);
const vector unsigned char dst2 = vec_ld(16, dst);
const vector unsigned char vdst = vec_perm(dst1, dst2, vec_lvsl(0, dst));
vector unsigned char fsum;
OP_U8_ALTIVEC(fsum, sum, vdst);
const vector unsigned char rsum = vec_perm(fsum, fsum, dstperm);
const vector unsigned char fdst1 = vec_sel(dst1, rsum, dstmask);
const vector unsigned char fdst2 = vec_sel(rsum, dst2, dstmask);
vec_st(fdst1, 0, dst);
vec_st(fdst2, 16, dst);
src += srcStride;
dst += dstStride;
}
POWERPC_PERF_STOP_COUNT(PREFIX_h264_qpel16_h_lowpass_num, 1);
}
void pix_diff :: processYUV_Altivec(imageStruct &image, imageStruct &right)
{
long h,w,width;
width = image.xsize/8;
//format is U Y V Y
union
{
//unsigned int i;
short elements[8];
//vector signed char v;
vector short v;
}shortBuffer;
vector signed short d, hiImage, loImage,hiRight, loRight;//, YRight, UVRight, YImage, UVImage, UVTemp, YTemp;
vector unsigned char zero = vec_splat_u8(0);
vector unsigned char *inData = (vector unsigned char*) image.data;
vector unsigned char *rightData = (vector unsigned char*) right.data;
shortBuffer.elements[0] = 128;
shortBuffer.elements[1] = 0;
shortBuffer.elements[2] = 128;
shortBuffer.elements[3] = 0;
shortBuffer.elements[4] = 128;
shortBuffer.elements[5] = 0;
shortBuffer.elements[6] = 128;
shortBuffer.elements[7] = 0;
//Load it into the vector unit
d = shortBuffer.v;
#ifndef PPC970
UInt32 prefetchSize = GetPrefetchConstant( 16, 1, 256 );
vec_dst( inData, prefetchSize, 0 );
vec_dst( rightData, prefetchSize, 1 );
#endif
for ( h=0; h<image.ysize; h++){
for (w=0; w<width; w++)
{
#ifndef PPC970
vec_dst( inData, prefetchSize, 0 );
vec_dst( rightData, prefetchSize, 1 );
#endif
//interleaved U Y V Y chars
//break out to unsigned shorts
hiImage = (vector signed short) vec_mergeh( zero, inData[0] );
loImage = (vector signed short) vec_mergel( zero, inData[0] );
hiRight = (vector signed short) vec_mergeh( zero, rightData[0] );
loRight = (vector signed short) vec_mergel( zero, rightData[0] );
//subtract the 128 offset for UV
hiImage = vec_subs(hiImage,d);
loImage = vec_subs(loImage,d);
hiRight = vec_subs(hiRight,d);
loRight = vec_subs(loRight,d);
hiImage = vec_subs(hiImage,hiRight);
loImage = vec_subs(loImage,loRight);
hiImage = vec_adds(hiImage,d);
loImage = vec_adds(loImage,d);
hiImage = vec_abs(hiImage);
loImage = vec_abs(loImage);
inData[0] = vec_packsu(hiImage, loImage);
inData++;
rightData++;
}
#ifndef PPC970
vec_dss( 0 );
vec_dss( 1 );
#endif
} /*end of working altivec function */
}
static void PREFIX_h264_qpel16_v_lowpass_altivec(uint8_t *dst,
const uint8_t *src,
int dstStride, int srcStride)
{
register int i;
LOAD_ZERO;
vec_u8 perm;
#if HAVE_BIGENDIAN
perm = vec_lvsl(0, src);
#endif
const vec_s16 v20ss = vec_sl(vec_splat_s16(5),vec_splat_u16(2));
const vec_u16 v5us = vec_splat_u16(5);
const vec_s16 v5ss = vec_splat_s16(5);
const vec_s16 v16ss = vec_sl(vec_splat_s16(1),vec_splat_u16(4));
const uint8_t *srcbis = src - (srcStride * 2);
const vec_u8 srcM2 = load_with_perm_vec(0, srcbis, perm);
srcbis += srcStride;
const vec_u8 srcM1 = load_with_perm_vec(0, srcbis, perm);
srcbis += srcStride;
const vec_u8 srcP0 = load_with_perm_vec(0, srcbis, perm);
srcbis += srcStride;
const vec_u8 srcP1 = load_with_perm_vec(0, srcbis, perm);
srcbis += srcStride;
const vec_u8 srcP2 = load_with_perm_vec(0, srcbis, perm);
srcbis += srcStride;
vec_s16 srcM2ssA = (vec_s16) VEC_MERGEH(zero_u8v, srcM2);
vec_s16 srcM2ssB = (vec_s16) VEC_MERGEL(zero_u8v, srcM2);
vec_s16 srcM1ssA = (vec_s16) VEC_MERGEH(zero_u8v, srcM1);
vec_s16 srcM1ssB = (vec_s16) VEC_MERGEL(zero_u8v, srcM1);
vec_s16 srcP0ssA = (vec_s16) VEC_MERGEH(zero_u8v, srcP0);
vec_s16 srcP0ssB = (vec_s16) VEC_MERGEL(zero_u8v, srcP0);
vec_s16 srcP1ssA = (vec_s16) VEC_MERGEH(zero_u8v, srcP1);
vec_s16 srcP1ssB = (vec_s16) VEC_MERGEL(zero_u8v, srcP1);
vec_s16 srcP2ssA = (vec_s16) VEC_MERGEH(zero_u8v, srcP2);
vec_s16 srcP2ssB = (vec_s16) VEC_MERGEL(zero_u8v, srcP2);
vec_s16 pp1A, pp1B, pp2A, pp2B, pp3A, pp3B,
psumA, psumB, sumA, sumB,
srcP3ssA, srcP3ssB,
sum1A, sum1B, sum2A, sum2B, sum3A, sum3B;
vec_u8 sum, fsum, srcP3;
for (i = 0 ; i < 16 ; i++) {
srcP3 = load_with_perm_vec(0, srcbis, perm);
srcbis += srcStride;
srcP3ssA = (vec_s16) VEC_MERGEH(zero_u8v, srcP3);
srcP3ssB = (vec_s16) VEC_MERGEL(zero_u8v, srcP3);
sum1A = vec_adds(srcP0ssA, srcP1ssA);
sum1B = vec_adds(srcP0ssB, srcP1ssB);
sum2A = vec_adds(srcM1ssA, srcP2ssA);
sum2B = vec_adds(srcM1ssB, srcP2ssB);
sum3A = vec_adds(srcM2ssA, srcP3ssA);
sum3B = vec_adds(srcM2ssB, srcP3ssB);
srcM2ssA = srcM1ssA;
srcM2ssB = srcM1ssB;
srcM1ssA = srcP0ssA;
srcM1ssB = srcP0ssB;
srcP0ssA = srcP1ssA;
srcP0ssB = srcP1ssB;
srcP1ssA = srcP2ssA;
srcP1ssB = srcP2ssB;
srcP2ssA = srcP3ssA;
srcP2ssB = srcP3ssB;
pp1A = vec_mladd(sum1A, v20ss, v16ss);
pp1B = vec_mladd(sum1B, v20ss, v16ss);
pp2A = vec_mladd(sum2A, v5ss, zero_s16v);
pp2B = vec_mladd(sum2B, v5ss, zero_s16v);
pp3A = vec_add(sum3A, pp1A);
pp3B = vec_add(sum3B, pp1B);
psumA = vec_sub(pp3A, pp2A);
psumB = vec_sub(pp3B, pp2B);
sumA = vec_sra(psumA, v5us);
sumB = vec_sra(psumB, v5us);
sum = vec_packsu(sumA, sumB);
ASSERT_ALIGNED(dst);
OP_U8_ALTIVEC(fsum, sum, vec_ld(0, dst));
vec_st(fsum, 0, dst);
dst += dstStride;
}
}
static void PREFIX_h264_qpel16_h_lowpass_altivec(uint8_t *dst,
const uint8_t *src,
int dstStride, int srcStride)
{
register int i;
LOAD_ZERO;
vec_u8 permM2, permM1, permP0, permP1, permP2, permP3;
const vec_s16 v5ss = vec_splat_s16(5);
const vec_u16 v5us = vec_splat_u16(5);
const vec_s16 v20ss = vec_sl(vec_splat_s16(5),vec_splat_u16(2));
const vec_s16 v16ss = vec_sl(vec_splat_s16(1),vec_splat_u16(4));
vec_u8 srcM2, srcM1, srcP0, srcP1, srcP2, srcP3;
register int align = ((((unsigned long)src) - 2) % 16);
vec_s16 srcP0A, srcP0B, srcP1A, srcP1B,
srcP2A, srcP2B, srcP3A, srcP3B,
srcM1A, srcM1B, srcM2A, srcM2B,
sum1A, sum1B, sum2A, sum2B, sum3A, sum3B,
pp1A, pp1B, pp2A, pp2B, pp3A, pp3B,
psumA, psumB, sumA, sumB;
vec_u8 sum, fsum;
#if HAVE_BIGENDIAN
permM2 = vec_lvsl(-2, src);
permM1 = vec_lvsl(-1, src);
permP0 = vec_lvsl(+0, src);
permP1 = vec_lvsl(+1, src);
permP2 = vec_lvsl(+2, src);
permP3 = vec_lvsl(+3, src);
#endif /* HAVE_BIGENDIAN */
for (i = 0 ; i < 16 ; i ++) {
load_alignment(src, align, permM2, permM1, permP0, permP1, permP2, permP3);
srcP0A = (vec_s16) VEC_MERGEH(zero_u8v, srcP0);
srcP0B = (vec_s16) VEC_MERGEL(zero_u8v, srcP0);
srcP1A = (vec_s16) VEC_MERGEH(zero_u8v, srcP1);
srcP1B = (vec_s16) VEC_MERGEL(zero_u8v, srcP1);
srcP2A = (vec_s16) VEC_MERGEH(zero_u8v, srcP2);
srcP2B = (vec_s16) VEC_MERGEL(zero_u8v, srcP2);
srcP3A = (vec_s16) VEC_MERGEH(zero_u8v, srcP3);
srcP3B = (vec_s16) VEC_MERGEL(zero_u8v, srcP3);
srcM1A = (vec_s16) VEC_MERGEH(zero_u8v, srcM1);
srcM1B = (vec_s16) VEC_MERGEL(zero_u8v, srcM1);
srcM2A = (vec_s16) VEC_MERGEH(zero_u8v, srcM2);
srcM2B = (vec_s16) VEC_MERGEL(zero_u8v, srcM2);
sum1A = vec_adds(srcP0A, srcP1A);
sum1B = vec_adds(srcP0B, srcP1B);
sum2A = vec_adds(srcM1A, srcP2A);
sum2B = vec_adds(srcM1B, srcP2B);
sum3A = vec_adds(srcM2A, srcP3A);
sum3B = vec_adds(srcM2B, srcP3B);
pp1A = vec_mladd(sum1A, v20ss, v16ss);
pp1B = vec_mladd(sum1B, v20ss, v16ss);
pp2A = vec_mladd(sum2A, v5ss, zero_s16v);
pp2B = vec_mladd(sum2B, v5ss, zero_s16v);
pp3A = vec_add(sum3A, pp1A);
pp3B = vec_add(sum3B, pp1B);
psumA = vec_sub(pp3A, pp2A);
psumB = vec_sub(pp3B, pp2B);
sumA = vec_sra(psumA, v5us);
sumB = vec_sra(psumB, v5us);
sum = vec_packsu(sumA, sumB);
ASSERT_ALIGNED(dst);
OP_U8_ALTIVEC(fsum, sum, vec_ld(0, dst));
vec_st(fsum, 0, dst);
src += srcStride;
dst += dstStride;
}
}
tmp1 = vec_perm(table[6], table[7], tmpIndex);
stmp1 = vec_perm(slope_cos[6], slope_cos[7], tmpIndex);
select = (vector unsigned short)vec_cmpgt(PerIndex, (((vector unsigned char){95,95,95,95,95,95,95,95,95,95,95,95,95,95,95,95})) );
tmp3 = vec_sel(tmp0, tmp1, select);
stmp3 = vec_sel(stmp0, stmp1, select);
select = (vector unsigned short)vec_cmpgt(PerIndex, (((vector unsigned char){63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63})) );
table1 = vec_sel(tmp2, tmp3, select);
slope1 = vec_sel(stmp2, stmp3, select);
L_tmp0 = vec_sra(vec_mule(slope0, offset0), (((vector unsigned int){12,12,12,12})) );
L_tmp1 = vec_sra(vec_mulo(slope0, offset0), (((vector unsigned int){12,12,12,12})) );
L_tmp2 = vec_sra(vec_mule(slope1, offset1), (((vector unsigned int){12,12,12,12})) );
L_tmp3 = vec_sra(vec_mulo(slope1, offset1), (((vector unsigned int){12,12,12,12})) );
tmp0 = vec_packs(L_tmp0, L_tmp2);
tmp1 = vec_packs(L_tmp1, L_tmp3);
tmp2 = vec_mergeh(tmp0, tmp1);
tmp3 = vec_mergel(tmp0, tmp1);
lspq[0] = vec_adds(table0, tmp2);
lspq[1] = vec_adds(table1, tmp3);
return;
}
/* this code assume stride % 16 == 0 */
static void PREFIX_h264_qpel16_v_lowpass_altivec(uint8_t * dst, uint8_t * src, int dstStride, int srcStride) {
POWERPC_PERF_DECLARE(PREFIX_h264_qpel16_v_lowpass_num, 1);
POWERPC_PERF_START_COUNT(PREFIX_h264_qpel16_v_lowpass_num, 1);
register int i;
const vector signed int vzero = vec_splat_s32(0);
const vector unsigned char perm = vec_lvsl(0, src);
const vector signed short v20ss = (const vector signed short)AVV(20);
const vector unsigned short v5us = vec_splat_u16(5);
const vector signed short v5ss = vec_splat_s16(5);
const vector signed short v16ss = (const vector signed short)AVV(16);
const vector unsigned char dstperm = vec_lvsr(0, dst);
const vector unsigned char neg1 = (const vector unsigned char)vec_splat_s8(-1);
const vector unsigned char dstmask = vec_perm((const vector unsigned char)vzero, neg1, dstperm);
uint8_t *srcbis = src - (srcStride * 2);
const vector unsigned char srcM2a = vec_ld(0, srcbis);
const vector unsigned char srcM2b = vec_ld(16, srcbis);
const vector unsigned char srcM2 = vec_perm(srcM2a, srcM2b, perm);
srcbis += srcStride;
const vector unsigned char srcM1a = vec_ld(0, srcbis);
const vector unsigned char srcM1b = vec_ld(16, srcbis);
const vector unsigned char srcM1 = vec_perm(srcM1a, srcM1b, perm);
srcbis += srcStride;
const vector unsigned char srcP0a = vec_ld(0, srcbis);
const vector unsigned char srcP0b = vec_ld(16, srcbis);
const vector unsigned char srcP0 = vec_perm(srcP0a, srcP0b, perm);
srcbis += srcStride;
const vector unsigned char srcP1a = vec_ld(0, srcbis);
const vector unsigned char srcP1b = vec_ld(16, srcbis);
const vector unsigned char srcP1 = vec_perm(srcP1a, srcP1b, perm);
srcbis += srcStride;
const vector unsigned char srcP2a = vec_ld(0, srcbis);
const vector unsigned char srcP2b = vec_ld(16, srcbis);
const vector unsigned char srcP2 = vec_perm(srcP2a, srcP2b, perm);
srcbis += srcStride;
vector signed short srcM2ssA = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcM2);
vector signed short srcM2ssB = (vector signed short)vec_mergel((vector unsigned char)vzero, srcM2);
vector signed short srcM1ssA = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcM1);
vector signed short srcM1ssB = (vector signed short)vec_mergel((vector unsigned char)vzero, srcM1);
vector signed short srcP0ssA = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP0);
vector signed short srcP0ssB = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP0);
vector signed short srcP1ssA = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP1);
vector signed short srcP1ssB = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP1);
vector signed short srcP2ssA = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP2);
vector signed short srcP2ssB = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP2);
for (i = 0 ; i < 16 ; i++) {
const vector unsigned char srcP3a = vec_ld(0, srcbis);
const vector unsigned char srcP3b = vec_ld(16, srcbis);
const vector unsigned char srcP3 = vec_perm(srcP3a, srcP3b, perm);
const vector signed short srcP3ssA = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP3);
const vector signed short srcP3ssB = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP3);
srcbis += srcStride;
const vector signed short sum1A = vec_adds(srcP0ssA, srcP1ssA);
const vector signed short sum1B = vec_adds(srcP0ssB, srcP1ssB);
const vector signed short sum2A = vec_adds(srcM1ssA, srcP2ssA);
const vector signed short sum2B = vec_adds(srcM1ssB, srcP2ssB);
const vector signed short sum3A = vec_adds(srcM2ssA, srcP3ssA);
const vector signed short sum3B = vec_adds(srcM2ssB, srcP3ssB);
srcM2ssA = srcM1ssA;
srcM2ssB = srcM1ssB;
srcM1ssA = srcP0ssA;
srcM1ssB = srcP0ssB;
srcP0ssA = srcP1ssA;
srcP0ssB = srcP1ssB;
srcP1ssA = srcP2ssA;
srcP1ssB = srcP2ssB;
srcP2ssA = srcP3ssA;
srcP2ssB = srcP3ssB;
const vector signed short pp1A = vec_mladd(sum1A, v20ss, v16ss);
const vector signed short pp1B = vec_mladd(sum1B, v20ss, v16ss);
const vector signed short pp2A = vec_mladd(sum2A, v5ss, (vector signed short)vzero);
const vector signed short pp2B = vec_mladd(sum2B, v5ss, (vector signed short)vzero);
const vector signed short pp3A = vec_add(sum3A, pp1A);
const vector signed short pp3B = vec_add(sum3B, pp1B);
const vector signed short psumA = vec_sub(pp3A, pp2A);
const vector signed short psumB = vec_sub(pp3B, pp2B);
const vector signed short sumA = vec_sra(psumA, v5us);
const vector signed short sumB = vec_sra(psumB, v5us);
const vector unsigned char sum = vec_packsu(sumA, sumB);
const vector unsigned char dst1 = vec_ld(0, dst);
const vector unsigned char dst2 = vec_ld(16, dst);
const vector unsigned char vdst = vec_perm(dst1, dst2, vec_lvsl(0, dst));
vector unsigned char fsum;
OP_U8_ALTIVEC(fsum, sum, vdst);
const vector unsigned char rsum = vec_perm(fsum, fsum, dstperm);
const vector unsigned char fdst1 = vec_sel(dst1, rsum, dstmask);
const vector unsigned char fdst2 = vec_sel(rsum, dst2, dstmask);
vec_st(fdst1, 0, dst);
vec_st(fdst2, 16, dst);
dst += dstStride;
}
POWERPC_PERF_STOP_COUNT(PREFIX_h264_qpel16_v_lowpass_num, 1);
}
static void PREFIX_h264_qpel16_v_lowpass_altivec(uint8_t * dst, uint8_t * src, int dstStride, int srcStride) {
register int i;
LOAD_ZERO;
const vec_u8 perm = vec_lvsl(0, src);
const vec_s16 v20ss = vec_sl(vec_splat_s16(5),vec_splat_u16(2));
const vec_u16 v5us = vec_splat_u16(5);
const vec_s16 v5ss = vec_splat_s16(5);
const vec_s16 v16ss = vec_sl(vec_splat_s16(1),vec_splat_u16(4));
uint8_t *srcbis = src - (srcStride * 2);
const vec_u8 srcM2a = vec_ld(0, srcbis);
const vec_u8 srcM2b = vec_ld(16, srcbis);
const vec_u8 srcM2 = vec_perm(srcM2a, srcM2b, perm);
//srcbis += srcStride;
const vec_u8 srcM1a = vec_ld(0, srcbis += srcStride);
const vec_u8 srcM1b = vec_ld(16, srcbis);
const vec_u8 srcM1 = vec_perm(srcM1a, srcM1b, perm);
//srcbis += srcStride;
const vec_u8 srcP0a = vec_ld(0, srcbis += srcStride);
const vec_u8 srcP0b = vec_ld(16, srcbis);
const vec_u8 srcP0 = vec_perm(srcP0a, srcP0b, perm);
//srcbis += srcStride;
const vec_u8 srcP1a = vec_ld(0, srcbis += srcStride);
const vec_u8 srcP1b = vec_ld(16, srcbis);
const vec_u8 srcP1 = vec_perm(srcP1a, srcP1b, perm);
//srcbis += srcStride;
const vec_u8 srcP2a = vec_ld(0, srcbis += srcStride);
const vec_u8 srcP2b = vec_ld(16, srcbis);
const vec_u8 srcP2 = vec_perm(srcP2a, srcP2b, perm);
//srcbis += srcStride;
vec_s16 srcM2ssA = (vec_s16) vec_mergeh(zero_u8v, srcM2);
vec_s16 srcM2ssB = (vec_s16) vec_mergel(zero_u8v, srcM2);
vec_s16 srcM1ssA = (vec_s16) vec_mergeh(zero_u8v, srcM1);
vec_s16 srcM1ssB = (vec_s16) vec_mergel(zero_u8v, srcM1);
vec_s16 srcP0ssA = (vec_s16) vec_mergeh(zero_u8v, srcP0);
vec_s16 srcP0ssB = (vec_s16) vec_mergel(zero_u8v, srcP0);
vec_s16 srcP1ssA = (vec_s16) vec_mergeh(zero_u8v, srcP1);
vec_s16 srcP1ssB = (vec_s16) vec_mergel(zero_u8v, srcP1);
vec_s16 srcP2ssA = (vec_s16) vec_mergeh(zero_u8v, srcP2);
vec_s16 srcP2ssB = (vec_s16) vec_mergel(zero_u8v, srcP2);
vec_s16 pp1A, pp1B, pp2A, pp2B, pp3A, pp3B,
psumA, psumB, sumA, sumB,
srcP3ssA, srcP3ssB,
sum1A, sum1B, sum2A, sum2B, sum3A, sum3B;
vec_u8 sum, fsum, srcP3a, srcP3b, srcP3;
for (i = 0 ; i < 16 ; i++) {
srcP3a = vec_ld(0, srcbis += srcStride);
srcP3b = vec_ld(16, srcbis);
srcP3 = vec_perm(srcP3a, srcP3b, perm);
srcP3ssA = (vec_s16) vec_mergeh(zero_u8v, srcP3);
srcP3ssB = (vec_s16) vec_mergel(zero_u8v, srcP3);
//srcbis += srcStride;
sum1A = vec_adds(srcP0ssA, srcP1ssA);
sum1B = vec_adds(srcP0ssB, srcP1ssB);
sum2A = vec_adds(srcM1ssA, srcP2ssA);
sum2B = vec_adds(srcM1ssB, srcP2ssB);
sum3A = vec_adds(srcM2ssA, srcP3ssA);
sum3B = vec_adds(srcM2ssB, srcP3ssB);
srcM2ssA = srcM1ssA;
srcM2ssB = srcM1ssB;
srcM1ssA = srcP0ssA;
srcM1ssB = srcP0ssB;
srcP0ssA = srcP1ssA;
srcP0ssB = srcP1ssB;
srcP1ssA = srcP2ssA;
srcP1ssB = srcP2ssB;
srcP2ssA = srcP3ssA;
srcP2ssB = srcP3ssB;
pp1A = vec_mladd(sum1A, v20ss, v16ss);
pp1B = vec_mladd(sum1B, v20ss, v16ss);
pp2A = vec_mladd(sum2A, v5ss, zero_s16v);
pp2B = vec_mladd(sum2B, v5ss, zero_s16v);
pp3A = vec_add(sum3A, pp1A);
pp3B = vec_add(sum3B, pp1B);
psumA = vec_sub(pp3A, pp2A);
psumB = vec_sub(pp3B, pp2B);
sumA = vec_sra(psumA, v5us);
sumB = vec_sra(psumB, v5us);
sum = vec_packsu(sumA, sumB);
ASSERT_ALIGNED(dst);
OP_U8_ALTIVEC(fsum, sum, vec_ld(0, dst));
vec_st(fsum, 0, dst);
dst += dstStride;
}
}
/* this code assume stride % 16 == 0 *and* tmp is properly aligned */
static void PREFIX_h264_qpel16_hv_lowpass_altivec(uint8_t * dst, int16_t * tmp, uint8_t * src, int dstStride, int tmpStride, int srcStride) {
POWERPC_PERF_DECLARE(PREFIX_h264_qpel16_hv_lowpass_num, 1);
POWERPC_PERF_START_COUNT(PREFIX_h264_qpel16_hv_lowpass_num, 1);
register int i;
const vector signed int vzero = vec_splat_s32(0);
const vector unsigned char permM2 = vec_lvsl(-2, src);
const vector unsigned char permM1 = vec_lvsl(-1, src);
const vector unsigned char permP0 = vec_lvsl(+0, src);
const vector unsigned char permP1 = vec_lvsl(+1, src);
const vector unsigned char permP2 = vec_lvsl(+2, src);
const vector unsigned char permP3 = vec_lvsl(+3, src);
const vector signed short v20ss = (const vector signed short)AVV(20);
const vector unsigned int v10ui = vec_splat_u32(10);
const vector signed short v5ss = vec_splat_s16(5);
const vector signed short v1ss = vec_splat_s16(1);
const vector signed int v512si = (const vector signed int)AVV(512);
const vector unsigned int v16ui = (const vector unsigned int)AVV(16);
register int align = ((((unsigned long)src) - 2) % 16);
src -= (2 * srcStride);
for (i = 0 ; i < 21 ; i ++) {
vector unsigned char srcM2, srcM1, srcP0, srcP1, srcP2, srcP3;
vector unsigned char srcR1 = vec_ld(-2, src);
vector unsigned char srcR2 = vec_ld(14, src);
switch (align) {
default: {
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = vec_perm(srcR1, srcR2, permP2);
srcP3 = vec_perm(srcR1, srcR2, permP3);
} break;
case 11: {
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = vec_perm(srcR1, srcR2, permP2);
srcP3 = srcR2;
} break;
case 12: {
vector unsigned char srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = srcR2;
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 13: {
vector unsigned char srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = srcR2;
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 14: {
vector unsigned char srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = srcR2;
srcP1 = vec_perm(srcR2, srcR3, permP1);
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 15: {
vector unsigned char srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = srcR2;
srcP0 = vec_perm(srcR2, srcR3, permP0);
srcP1 = vec_perm(srcR2, srcR3, permP1);
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
}
const vector signed short srcP0A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP0);
const vector signed short srcP0B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP0);
const vector signed short srcP1A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP1);
const vector signed short srcP1B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP1);
const vector signed short srcP2A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP2);
const vector signed short srcP2B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP2);
const vector signed short srcP3A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcP3);
const vector signed short srcP3B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcP3);
const vector signed short srcM1A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcM1);
const vector signed short srcM1B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcM1);
const vector signed short srcM2A = (vector signed short)vec_mergeh((vector unsigned char)vzero, srcM2);
const vector signed short srcM2B = (vector signed short)vec_mergel((vector unsigned char)vzero, srcM2);
const vector signed short sum1A = vec_adds(srcP0A, srcP1A);
const vector signed short sum1B = vec_adds(srcP0B, srcP1B);
const vector signed short sum2A = vec_adds(srcM1A, srcP2A);
const vector signed short sum2B = vec_adds(srcM1B, srcP2B);
const vector signed short sum3A = vec_adds(srcM2A, srcP3A);
const vector signed short sum3B = vec_adds(srcM2B, srcP3B);
const vector signed short pp1A = vec_mladd(sum1A, v20ss, sum3A);
const vector signed short pp1B = vec_mladd(sum1B, v20ss, sum3B);
const vector signed short pp2A = vec_mladd(sum2A, v5ss, (vector signed short)vzero);
const vector signed short pp2B = vec_mladd(sum2B, v5ss, (vector signed short)vzero);
const vector signed short psumA = vec_sub(pp1A, pp2A);
const vector signed short psumB = vec_sub(pp1B, pp2B);
vec_st(psumA, 0, tmp);
vec_st(psumB, 16, tmp);
src += srcStride;
tmp += tmpStride; /* int16_t*, and stride is 16, so it's OK here */
}
const vector unsigned char dstperm = vec_lvsr(0, dst);
const vector unsigned char neg1 = (const vector unsigned char)vec_splat_s8(-1);
const vector unsigned char dstmask = vec_perm((const vector unsigned char)vzero, neg1, dstperm);
const vector unsigned char mperm = (const vector unsigned char)
AVV(0x00, 0x08, 0x01, 0x09, 0x02, 0x0A, 0x03, 0x0B,
0x04, 0x0C, 0x05, 0x0D, 0x06, 0x0E, 0x07, 0x0F);
int16_t *tmpbis = tmp - (tmpStride * 21);
vector signed short tmpM2ssA = vec_ld(0, tmpbis);
vector signed short tmpM2ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
vector signed short tmpM1ssA = vec_ld(0, tmpbis);
vector signed short tmpM1ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
vector signed short tmpP0ssA = vec_ld(0, tmpbis);
vector signed short tmpP0ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
vector signed short tmpP1ssA = vec_ld(0, tmpbis);
vector signed short tmpP1ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
vector signed short tmpP2ssA = vec_ld(0, tmpbis);
vector signed short tmpP2ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
for (i = 0 ; i < 16 ; i++) {
const vector signed short tmpP3ssA = vec_ld(0, tmpbis);
const vector signed short tmpP3ssB = vec_ld(16, tmpbis);
tmpbis += tmpStride;
const vector signed short sum1A = vec_adds(tmpP0ssA, tmpP1ssA);
const vector signed short sum1B = vec_adds(tmpP0ssB, tmpP1ssB);
const vector signed short sum2A = vec_adds(tmpM1ssA, tmpP2ssA);
const vector signed short sum2B = vec_adds(tmpM1ssB, tmpP2ssB);
const vector signed short sum3A = vec_adds(tmpM2ssA, tmpP3ssA);
const vector signed short sum3B = vec_adds(tmpM2ssB, tmpP3ssB);
tmpM2ssA = tmpM1ssA;
tmpM2ssB = tmpM1ssB;
tmpM1ssA = tmpP0ssA;
tmpM1ssB = tmpP0ssB;
tmpP0ssA = tmpP1ssA;
tmpP0ssB = tmpP1ssB;
tmpP1ssA = tmpP2ssA;
tmpP1ssB = tmpP2ssB;
tmpP2ssA = tmpP3ssA;
tmpP2ssB = tmpP3ssB;
const vector signed int pp1Ae = vec_mule(sum1A, v20ss);
const vector signed int pp1Ao = vec_mulo(sum1A, v20ss);
const vector signed int pp1Be = vec_mule(sum1B, v20ss);
const vector signed int pp1Bo = vec_mulo(sum1B, v20ss);
const vector signed int pp2Ae = vec_mule(sum2A, v5ss);
const vector signed int pp2Ao = vec_mulo(sum2A, v5ss);
const vector signed int pp2Be = vec_mule(sum2B, v5ss);
const vector signed int pp2Bo = vec_mulo(sum2B, v5ss);
const vector signed int pp3Ae = vec_sra((vector signed int)sum3A, v16ui);
const vector signed int pp3Ao = vec_mulo(sum3A, v1ss);
const vector signed int pp3Be = vec_sra((vector signed int)sum3B, v16ui);
const vector signed int pp3Bo = vec_mulo(sum3B, v1ss);
const vector signed int pp1cAe = vec_add(pp1Ae, v512si);
const vector signed int pp1cAo = vec_add(pp1Ao, v512si);
const vector signed int pp1cBe = vec_add(pp1Be, v512si);
const vector signed int pp1cBo = vec_add(pp1Bo, v512si);
const vector signed int pp32Ae = vec_sub(pp3Ae, pp2Ae);
const vector signed int pp32Ao = vec_sub(pp3Ao, pp2Ao);
const vector signed int pp32Be = vec_sub(pp3Be, pp2Be);
const vector signed int pp32Bo = vec_sub(pp3Bo, pp2Bo);
const vector signed int sumAe = vec_add(pp1cAe, pp32Ae);
const vector signed int sumAo = vec_add(pp1cAo, pp32Ao);
const vector signed int sumBe = vec_add(pp1cBe, pp32Be);
const vector signed int sumBo = vec_add(pp1cBo, pp32Bo);
const vector signed int ssumAe = vec_sra(sumAe, v10ui);
const vector signed int ssumAo = vec_sra(sumAo, v10ui);
const vector signed int ssumBe = vec_sra(sumBe, v10ui);
const vector signed int ssumBo = vec_sra(sumBo, v10ui);
const vector signed short ssume = vec_packs(ssumAe, ssumBe);
const vector signed short ssumo = vec_packs(ssumAo, ssumBo);
const vector unsigned char sumv = vec_packsu(ssume, ssumo);
const vector unsigned char sum = vec_perm(sumv, sumv, mperm);
const vector unsigned char dst1 = vec_ld(0, dst);
const vector unsigned char dst2 = vec_ld(16, dst);
const vector unsigned char vdst = vec_perm(dst1, dst2, vec_lvsl(0, dst));
vector unsigned char fsum;
OP_U8_ALTIVEC(fsum, sum, vdst);
const vector unsigned char rsum = vec_perm(fsum, fsum, dstperm);
const vector unsigned char fdst1 = vec_sel(dst1, rsum, dstmask);
const vector unsigned char fdst2 = vec_sel(rsum, dst2, dstmask);
vec_st(fdst1, 0, dst);
vec_st(fdst2, 16, dst);
dst += dstStride;
}
POWERPC_PERF_STOP_COUNT(PREFIX_h264_qpel16_hv_lowpass_num, 1);
}
static void PREFIX_h264_qpel16_h_lowpass_altivec(uint8_t * dst, uint8_t * src, int dstStride, int srcStride) {
register int i;
LOAD_ZERO;
const vec_u8 permM2 = vec_lvsl(-2, src);
const vec_u8 permM1 = vec_lvsl(-1, src);
const vec_u8 permP0 = vec_lvsl(+0, src);
const vec_u8 permP1 = vec_lvsl(+1, src);
const vec_u8 permP2 = vec_lvsl(+2, src);
const vec_u8 permP3 = vec_lvsl(+3, src);
const vec_s16 v5ss = vec_splat_s16(5);
const vec_u16 v5us = vec_splat_u16(5);
const vec_s16 v20ss = vec_sl(vec_splat_s16(5),vec_splat_u16(2));
const vec_s16 v16ss = vec_sl(vec_splat_s16(1),vec_splat_u16(4));
vec_u8 srcM2, srcM1, srcP0, srcP1, srcP2, srcP3;
register int align = ((((unsigned long)src) - 2) % 16);
vec_s16 srcP0A, srcP0B, srcP1A, srcP1B,
srcP2A, srcP2B, srcP3A, srcP3B,
srcM1A, srcM1B, srcM2A, srcM2B,
sum1A, sum1B, sum2A, sum2B, sum3A, sum3B,
pp1A, pp1B, pp2A, pp2B, pp3A, pp3B,
psumA, psumB, sumA, sumB;
vec_u8 sum, fsum;
for (i = 0 ; i < 16 ; i ++) {
vec_u8 srcR1 = vec_ld(-2, src);
vec_u8 srcR2 = vec_ld(14, src);
switch (align) {
default: {
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = vec_perm(srcR1, srcR2, permP2);
srcP3 = vec_perm(srcR1, srcR2, permP3);
} break;
case 11: {
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = vec_perm(srcR1, srcR2, permP2);
srcP3 = srcR2;
} break;
case 12: {
vec_u8 srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = vec_perm(srcR1, srcR2, permP1);
srcP2 = srcR2;
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 13: {
vec_u8 srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = vec_perm(srcR1, srcR2, permP0);
srcP1 = srcR2;
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 14: {
vec_u8 srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = vec_perm(srcR1, srcR2, permM1);
srcP0 = srcR2;
srcP1 = vec_perm(srcR2, srcR3, permP1);
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
case 15: {
vec_u8 srcR3 = vec_ld(30, src);
srcM2 = vec_perm(srcR1, srcR2, permM2);
srcM1 = srcR2;
srcP0 = vec_perm(srcR2, srcR3, permP0);
srcP1 = vec_perm(srcR2, srcR3, permP1);
srcP2 = vec_perm(srcR2, srcR3, permP2);
srcP3 = vec_perm(srcR2, srcR3, permP3);
} break;
}
srcP0A = (vec_s16) vec_mergeh(zero_u8v, srcP0);
srcP0B = (vec_s16) vec_mergel(zero_u8v, srcP0);
srcP1A = (vec_s16) vec_mergeh(zero_u8v, srcP1);
srcP1B = (vec_s16) vec_mergel(zero_u8v, srcP1);
srcP2A = (vec_s16) vec_mergeh(zero_u8v, srcP2);
srcP2B = (vec_s16) vec_mergel(zero_u8v, srcP2);
srcP3A = (vec_s16) vec_mergeh(zero_u8v, srcP3);
srcP3B = (vec_s16) vec_mergel(zero_u8v, srcP3);
srcM1A = (vec_s16) vec_mergeh(zero_u8v, srcM1);
srcM1B = (vec_s16) vec_mergel(zero_u8v, srcM1);
srcM2A = (vec_s16) vec_mergeh(zero_u8v, srcM2);
srcM2B = (vec_s16) vec_mergel(zero_u8v, srcM2);
sum1A = vec_adds(srcP0A, srcP1A);
sum1B = vec_adds(srcP0B, srcP1B);
sum2A = vec_adds(srcM1A, srcP2A);
sum2B = vec_adds(srcM1B, srcP2B);
sum3A = vec_adds(srcM2A, srcP3A);
sum3B = vec_adds(srcM2B, srcP3B);
pp1A = vec_mladd(sum1A, v20ss, v16ss);
pp1B = vec_mladd(sum1B, v20ss, v16ss);
pp2A = vec_mladd(sum2A, v5ss, zero_s16v);
pp2B = vec_mladd(sum2B, v5ss, zero_s16v);
pp3A = vec_add(sum3A, pp1A);
pp3B = vec_add(sum3B, pp1B);
psumA = vec_sub(pp3A, pp2A);
psumB = vec_sub(pp3B, pp2B);
sumA = vec_sra(psumA, v5us);
sumB = vec_sra(psumB, v5us);
sum = vec_packsu(sumA, sumB);
ASSERT_ALIGNED(dst);
OP_U8_ALTIVEC(fsum, sum, vec_ld(0, dst));
vec_st(fsum, 0, dst);
src += srcStride;
dst += dstStride;
}
}
static force_inline vector unsigned int
pix_add (vector unsigned int a, vector unsigned int b)
{
return (vector unsigned int)vec_adds ((vector unsigned char)a,
(vector unsigned char)b);
}
