int kernel(lwp_functions* pf, void* params, void* inout, unsigned int iter, unsigned int n) { Ternary_params* p = (Ternary_params*)params; switch (p->cmd) { case AM: { int length = p->length / 4; vector float *a = (vector float *)inout; vector float *b = a + length; vector float *c = a + 2 * length; unsigned int i; for (i = 0; i != length; ++i, ++a, ++b, ++c) *a = spu_mul(spu_add(*a, *b), *c); return 0; } case MA: { int length = p->length / 4; vector float *a = (vector float *)inout; vector float *b = a + length; vector float *c = a + 2 * length; unsigned int i; for (i = 0; i != length; ++i, ++a, ++b, ++c) *a = spu_madd(*a, *b, *c); return 0; } case CAM: { static vector unsigned char lo = (vector unsigned char) { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}; static vector unsigned char hi = (vector unsigned char) { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31}; int length = p->length / 4; float *a = (float *)inout; float *b = a + 8 * length; float *c = a + 16 * length; unsigned int i; // (a + b) * c: // r.r = (a.r+b.r)*c.r - (a.i+b.i)*c.i // r.i = (a.r+b.r)*c.i + (a.i+b.i)*c.r for (i = 0; i != length; ++i, a+=8, b+=8, c+=8) { vector float av = {*a, *(a+2), *(a+4), *(a+6)}; // a.r vector float bv = {*b, *(b+2), *(b+4), *(b+6)}; // b.r vector float cv = {*c, *(c+2), *(c+4), *(c+6)}; // c.r vector float dv = {*(a+1), *(a+3), *(a+5), *(a+7)}; // a.i vector float ev = {*(b+1), *(b+3), *(b+5), *(b+7)}; // b.i vector float fv = {*(c+1), *(c+3), *(c+5), *(c+7)}; // c.i vector float trv = spu_add(av, bv); // a.r+b.r vector float tiv = spu_add(dv, ev); // a.i+b.i vector float sv = spu_mul(trv, cv); // (a.r+b.r)*c.r vector float tv = spu_mul(trv, fv); // (a.r+b.r)*c.i vector float real = spu_nmsub(tiv, fv, sv); // r.r vector float imag = spu_madd(tiv, cv, tv); // r.i // interleave result *(vector float *)a = spu_shuffle(real, imag, lo); *(vector float *)(a+4) = spu_shuffle(real, imag, hi); } return 0; } case CMA: { static vector unsigned char lo = (vector unsigned char) { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}; static vector unsigned char hi = (vector unsigned char) { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31}; int length = p->length / 4; float *a = (float *)inout; float *b = a + 8 * length; float *c = a + 16 * length; unsigned int i; // a * b + c: // r.r = a.r*b.r + c.r - a.i*b.i // r.i = a.r*b.i + c.i + a.i*b.r for (i = 0; i != length; ++i, a+=8, b+=8, c+=8) { vector float av = {*a, *(a+2), *(a+4), *(a+6)}; // a.r vector float bv = {*b, *(b+2), *(b+4), *(b+6)}; // b.r vector float cv = {*c, *(c+2), *(c+4), *(c+6)}; // c.r vector float dv = {*(a+1), *(a+3), *(a+5), *(a+7)}; // a.i vector float ev = {*(b+1), *(b+3), *(b+5), *(b+7)}; // b.i vector float fv = {*(c+1), *(c+3), *(c+5), *(c+7)}; // c.i vector float real = spu_nmsub(dv, ev, spu_madd(av, bv, cv)); // r.r vector float imag = spu_madd(dv, bv, spu_madd(av, ev, fv)); // r.i // interleave result *(vector float *)a = spu_shuffle(real, imag, lo); *(vector float *)(a+4) = spu_shuffle(real, imag, hi); } return 0; } case ZAM: { int length = p->length / 4; float *a_re = (float *)inout; float *a_im = a_re + 4 * length; float *b_re = a_re + 8 * length; float *b_im = a_re + 12 * length; float *c_re = a_re + 16 * length; float *c_im = a_re + 20 * length; unsigned int i; // (a + b) * c: // r.r = (a.r+b.r)*c.r - (a.i+b.i)*c.i // r.i = (a.r+b.r)*c.i + (a.i+b.i)*c.r for (i = 0; i != length; ++i, a_re+=4, b_re+=4, c_re+=4, a_im+=4, b_im+=4, c_im+=4) { vector float *av = (vector float *)a_re; vector float *bv = (vector float *)b_re; vector float *cv = (vector float *)c_re; vector float *dv = (vector float *)a_im; vector float *ev = (vector float *)b_im; vector float *fv = (vector float *)c_im; vector float trv = spu_add(*av, *bv); // a.r+b.r vector float tiv = spu_add(*dv, *ev); // a.i+b.i vector float sv = spu_mul(trv, *cv); // (a.r+b.r)*c.r vector float tv = spu_mul(trv, *fv); // (a.r+b.r)*c.i *av = spu_nmsub(tiv, *fv, sv); // r.r *dv = spu_madd(tiv, *cv, tv); // r.i } return 0; } case ZMA: { int length = p->length / 4; float *a_re = (float *)inout; float *a_im = a_re + 4 * length; float *b_re = a_re + 8 * length; float *b_im = a_re + 12 * length; float *c_re = a_re + 16 * length; float *c_im = a_re + 20 * length; unsigned int i; // a * b + c: // r.r = a.r*b.r + c.r - a.i*b.i // r.i = a.r*b.i + c.i + a.i*b.r for (i = 0; i != length; ++i, a_re+=4, b_re+=4, c_re+=4, a_im+=4, b_im+=4, c_im+=4) { vector float *av = (vector float *)a_re; vector float *bv = (vector float *)b_re; vector float *cv = (vector float *)c_re; vector float *dv = (vector float *)a_im; vector float *ev = (vector float *)b_im; vector float *fv = (vector float *)c_im; vector float tmp = spu_nmsub(*dv, *ev, spu_madd(*av, *bv, *cv)); *dv = spu_madd(*dv, *bv, spu_madd(*av, *ev, *fv)); *av = tmp; } return 0; } } return 1; }
vector double __divv2df3 (vector double a_in, vector double b_in) { /* Variables */ vec_int4 exp, exp_bias; vec_uint4 no_underflow, overflow; vec_float4 mant_bf, inv_bf; vec_ullong2 exp_a, exp_b; vec_ullong2 a_nan, a_zero, a_inf, a_denorm, a_denorm0; vec_ullong2 b_nan, b_zero, b_inf, b_denorm, b_denorm0; vec_ullong2 nan; vec_uint4 a_exp, b_exp; vec_ullong2 a_mant_0, b_mant_0; vec_ullong2 a_exp_1s, b_exp_1s; vec_ullong2 sign_exp_mask; vec_double2 a, b; vec_double2 mant_a, mant_b, inv_b, q0, q1, q2, mult; /* Constants */ vec_uint4 exp_mask_u32 = spu_splats((unsigned int)0x7FF00000); vec_uchar16 splat_hi = (vec_uchar16) { 0,1,2,3, 0,1,2,3, 8, 9,10,11, 8,9,10,11 }; vec_uchar16 swap_32 = (vec_uchar16) { 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 }; vec_ullong2 exp_mask = spu_splats(0x7FF0000000000000ULL); vec_ullong2 sign_mask = spu_splats(0x8000000000000000ULL); vec_float4 onef = spu_splats(1.0f); vec_double2 one = spu_splats(1.0); vec_double2 exp_53 = (vec_double2)spu_splats(0x0350000000000000ULL); sign_exp_mask = spu_or(sign_mask, exp_mask); /* Extract the floating point components from each of the operands including * exponent and mantissa. */ a_exp = (vec_uint4)spu_and((vec_uint4)a_in, exp_mask_u32); a_exp = spu_shuffle(a_exp, a_exp, splat_hi); b_exp = (vec_uint4)spu_and((vec_uint4)b_in, exp_mask_u32); b_exp = spu_shuffle(b_exp, b_exp, splat_hi); a_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)a_in, sign_exp_mask), 0); a_mant_0 = spu_and(a_mant_0, spu_shuffle(a_mant_0, a_mant_0, swap_32)); b_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)b_in, sign_exp_mask), 0); b_mant_0 = spu_and(b_mant_0, spu_shuffle(b_mant_0, b_mant_0, swap_32)); a_exp_1s = (vec_ullong2)spu_cmpeq(a_exp, exp_mask_u32); b_exp_1s = (vec_ullong2)spu_cmpeq(b_exp, exp_mask_u32); /* Identify all possible special values that must be accommodated including: * +-denorm, +-0, +-infinity, and NaNs. */ a_denorm0= (vec_ullong2)spu_cmpeq(a_exp, 0); a_nan = spu_andc(a_exp_1s, a_mant_0); a_zero = spu_and (a_denorm0, a_mant_0); a_inf = spu_and (a_exp_1s, a_mant_0); a_denorm = spu_andc(a_denorm0, a_zero); b_denorm0= (vec_ullong2)spu_cmpeq(b_exp, 0); b_nan = spu_andc(b_exp_1s, b_mant_0); b_zero = spu_and (b_denorm0, b_mant_0); b_inf = spu_and (b_exp_1s, b_mant_0); b_denorm = spu_andc(b_denorm0, b_zero); /* Scale denorm inputs to into normalized numbers by conditionally scaling the * input parameters. */ a = spu_sub(spu_or(a_in, exp_53), spu_sel(exp_53, a_in, sign_mask)); a = spu_sel(a_in, a, a_denorm); b = spu_sub(spu_or(b_in, exp_53), spu_sel(exp_53, b_in, sign_mask)); b = spu_sel(b_in, b, b_denorm); /* Extract the divisor and dividend exponent and force parameters into the signed * range [1.0,2.0) or [-1.0,2.0). */ exp_a = spu_and((vec_ullong2)a, exp_mask); exp_b = spu_and((vec_ullong2)b, exp_mask); mant_a = spu_sel(a, one, (vec_ullong2)exp_mask); mant_b = spu_sel(b, one, (vec_ullong2)exp_mask); /* Approximate the single reciprocal of b by using * the single precision reciprocal estimate followed by one * single precision iteration of Newton-Raphson. */ mant_bf = spu_roundtf(mant_b); inv_bf = spu_re(mant_bf); inv_bf = spu_madd(spu_nmsub(mant_bf, inv_bf, onef), inv_bf, inv_bf); /* Perform 2 more Newton-Raphson iterations in double precision. The * result (q1) is in the range (0.5, 2.0). */ inv_b = spu_extend(inv_bf); inv_b = spu_madd(spu_nmsub(mant_b, inv_b, one), inv_b, inv_b); q0 = spu_mul(mant_a, inv_b); q1 = spu_madd(spu_nmsub(mant_b, q0, mant_a), inv_b, q0); /* Determine the exponent correction factor that must be applied * to q1 by taking into account the exponent of the normalized inputs * and the scale factors that were applied to normalize them. */ exp = spu_rlmaska(spu_sub((vec_int4)exp_a, (vec_int4)exp_b), -20); exp = spu_add(exp, (vec_int4)spu_add(spu_and((vec_int4)a_denorm, -0x34), spu_and((vec_int4)b_denorm, 0x34))); /* Bias the quotient exponent depending on the sign of the exponent correction * factor so that a single multiplier will ensure the entire double precision * domain (including denorms) can be achieved. * * exp bias q1 adjust exp * ===== ======== ========== * positive 2^+65 -65 * negative 2^-64 +64 */ exp_bias = spu_xor(spu_rlmaska(exp, -31), 64); exp = spu_sub(exp, exp_bias); q1 = spu_sel(q1, (vec_double2)spu_add((vec_int4)q1, spu_sl(exp_bias, 20)), exp_mask); /* Compute a multiplier (mult) to applied to the quotient (q1) to produce the * expected result. On overflow, clamp the multiplier to the maximum non-infinite * number in case the rounding mode is not round-to-nearest. */ exp = spu_add(exp, 0x3FF); no_underflow = spu_cmpgt(exp, 0); overflow = spu_cmpgt(exp, 0x7FE); exp = spu_and(spu_sl(exp, 20), (vec_int4)no_underflow); exp = spu_and(exp, (vec_int4)exp_mask); mult = spu_sel((vec_double2)exp, (vec_double2)(spu_add((vec_uint4)exp_mask, -1)), (vec_ullong2)overflow); /* Handle special value conditions. These include: * * 1) IF either operand is a NaN OR both operands are 0 or INFINITY THEN a NaN * results. * 2) ELSE IF the dividend is an INFINITY OR the divisor is 0 THEN a INFINITY results. * 3) ELSE IF the dividend is 0 OR the divisor is INFINITY THEN a 0 results. */ mult = spu_andc(mult, (vec_double2)spu_or(a_zero, b_inf)); mult = spu_sel(mult, (vec_double2)exp_mask, spu_or(a_inf, b_zero)); nan = spu_or(a_nan, b_nan); nan = spu_or(nan, spu_and(a_zero, b_zero)); nan = spu_or(nan, spu_and(a_inf, b_inf)); mult = spu_or(mult, (vec_double2)nan); /* Scale the final quotient */ q2 = spu_mul(q1, mult); return (q2); }