static inline
vec_uint4 vec_sigma1(vec_uint4 x)
{
return spu_xor(spu_xor(vec_ROTR(17, x),
vec_ROTR(19, x)),
vec_SHR(10, x));
}
static inline
vec_uint4 vec_sigma0(vec_uint4 x)
{
return spu_xor(spu_xor(vec_ROTR( 7, x),
vec_ROTR(18, x)),
vec_SHR(3, x));
}
static inline
vec_uint4 vec_Sigma1(vec_uint4 x)
{
return spu_xor(spu_xor(vec_ROTR( 6, x),
vec_ROTR(11, x)),
vec_ROTR(25, x));
}
static inline
vec_uint4 vec_Sigma0(vec_uint4 x)
{
return spu_xor(spu_xor(vec_ROTR( 2, x),
vec_ROTR(13, x)),
vec_ROTR(22, x));
}
unsigned int
__mfc_multi_tag_reserve (unsigned int number_of_tags)
{
vector unsigned int table_copy;
vector unsigned int one = (vector unsigned int)
{ 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF };
vector unsigned int count_busy, is_valid;
vector unsigned int count_total;
vector unsigned int count_avail = (vector unsigned int) { 0, 0, 0, 0 };
vector unsigned int index = (vector unsigned int) { 0, 0, 0, 0 };
table_copy = __mfc_tag_table;
/* count_busy: number of consecutive busy tags
count_avail: number of consecutive free tags
table_copy: temporary copy of the tag table
count_total: sum of count_busy and count_avail
index: index of the current working tag */
do
{
table_copy = spu_sl (table_copy, count_avail);
count_busy = spu_cntlz (table_copy);
table_copy = spu_sl (table_copy, count_busy);
count_avail = spu_cntlz (spu_xor(table_copy, -1));
count_total = spu_add (count_busy, count_avail);
index = spu_add (index, count_total);
}
while (spu_extract (count_avail, 0) < number_of_tags
&& spu_extract (table_copy, 0) != 0);
index = spu_sub (index, count_avail);
/* is_valid is set to 0xFFFFFFFF if table_copy == 0, 0 otherwise. */
is_valid = spu_cmpeq (table_copy, 0);
index = spu_sel (index, is_valid, is_valid);
/* Now I need to actually mark the tags as used. */
table_copy = spu_sl (one, number_of_tags);
table_copy = spu_rl (table_copy, -number_of_tags - spu_extract (index, 0));
table_copy = spu_sel (table_copy, __mfc_tag_table, table_copy);
__mfc_tag_table = spu_sel (table_copy, __mfc_tag_table, is_valid);
return spu_extract (index, 0);
}
unsigned int
__mfc_multi_tag_release (unsigned int first_tag, unsigned int number_of_tags)
{
vector unsigned int table_copy, tmp, tmp1;
vector unsigned int one = (vector unsigned int)
{ 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF };
vector unsigned int is_invalid;
unsigned int last_tag;
vector unsigned int has_been_reserved;
last_tag = first_tag + number_of_tags;
table_copy = spu_sl (one, number_of_tags);
table_copy = spu_rl (table_copy, -last_tag);
table_copy = spu_xor (table_copy, -1);
/* Make sure the tags are in range and valid. */
tmp = spu_cmpgt (spu_promote(last_tag, 0), 32);
tmp1 = spu_cmpgt (spu_promote(number_of_tags, 0), 32);
is_invalid = spu_cmpgt (spu_promote(first_tag, 0), 31);
/* All bits are set to 1 if invalid, 0 if valid. */
is_invalid = spu_or (tmp, is_invalid);
is_invalid = spu_or (tmp1, is_invalid);
/* check whether these tags have been reserved */
tmp = spu_rlmask (one, (int)-number_of_tags);
tmp1 = spu_sl (__mfc_tag_table, first_tag);
has_been_reserved = spu_cmpgt(tmp1, tmp);
is_invalid = spu_or (has_been_reserved, is_invalid);
table_copy = spu_sel (__mfc_tag_table, table_copy, table_copy);
__mfc_tag_table = spu_sel (table_copy, __mfc_tag_table, is_invalid);
return spu_extract (is_invalid, 0);
}
vector double
__divv2df3 (vector double a_in, vector double b_in)
{
/* Variables */
vec_int4 exp, exp_bias;
vec_uint4 no_underflow, overflow;
vec_float4 mant_bf, inv_bf;
vec_ullong2 exp_a, exp_b;
vec_ullong2 a_nan, a_zero, a_inf, a_denorm, a_denorm0;
vec_ullong2 b_nan, b_zero, b_inf, b_denorm, b_denorm0;
vec_ullong2 nan;
vec_uint4 a_exp, b_exp;
vec_ullong2 a_mant_0, b_mant_0;
vec_ullong2 a_exp_1s, b_exp_1s;
vec_ullong2 sign_exp_mask;
vec_double2 a, b;
vec_double2 mant_a, mant_b, inv_b, q0, q1, q2, mult;
/* Constants */
vec_uint4 exp_mask_u32 = spu_splats((unsigned int)0x7FF00000);
vec_uchar16 splat_hi = (vec_uchar16) {
0,1,2,3, 0,1,2,3, 8, 9,10,11, 8,9,10,11
};
vec_uchar16 swap_32 = (vec_uchar16) {
4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11
};
vec_ullong2 exp_mask = spu_splats(0x7FF0000000000000ULL);
vec_ullong2 sign_mask = spu_splats(0x8000000000000000ULL);
vec_float4 onef = spu_splats(1.0f);
vec_double2 one = spu_splats(1.0);
vec_double2 exp_53 = (vec_double2)spu_splats(0x0350000000000000ULL);
sign_exp_mask = spu_or(sign_mask, exp_mask);
/* Extract the floating point components from each of the operands including
* exponent and mantissa.
*/
a_exp = (vec_uint4)spu_and((vec_uint4)a_in, exp_mask_u32);
a_exp = spu_shuffle(a_exp, a_exp, splat_hi);
b_exp = (vec_uint4)spu_and((vec_uint4)b_in, exp_mask_u32);
b_exp = spu_shuffle(b_exp, b_exp, splat_hi);
a_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)a_in, sign_exp_mask), 0);
a_mant_0 = spu_and(a_mant_0, spu_shuffle(a_mant_0, a_mant_0, swap_32));
b_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)b_in, sign_exp_mask), 0);
b_mant_0 = spu_and(b_mant_0, spu_shuffle(b_mant_0, b_mant_0, swap_32));
a_exp_1s = (vec_ullong2)spu_cmpeq(a_exp, exp_mask_u32);
b_exp_1s = (vec_ullong2)spu_cmpeq(b_exp, exp_mask_u32);
/* Identify all possible special values that must be accommodated including:
* +-denorm, +-0, +-infinity, and NaNs.
*/
a_denorm0= (vec_ullong2)spu_cmpeq(a_exp, 0);
a_nan = spu_andc(a_exp_1s, a_mant_0);
a_zero = spu_and (a_denorm0, a_mant_0);
a_inf = spu_and (a_exp_1s, a_mant_0);
a_denorm = spu_andc(a_denorm0, a_zero);
b_denorm0= (vec_ullong2)spu_cmpeq(b_exp, 0);
b_nan = spu_andc(b_exp_1s, b_mant_0);
b_zero = spu_and (b_denorm0, b_mant_0);
b_inf = spu_and (b_exp_1s, b_mant_0);
b_denorm = spu_andc(b_denorm0, b_zero);
/* Scale denorm inputs to into normalized numbers by conditionally scaling the
* input parameters.
*/
a = spu_sub(spu_or(a_in, exp_53), spu_sel(exp_53, a_in, sign_mask));
a = spu_sel(a_in, a, a_denorm);
b = spu_sub(spu_or(b_in, exp_53), spu_sel(exp_53, b_in, sign_mask));
b = spu_sel(b_in, b, b_denorm);
/* Extract the divisor and dividend exponent and force parameters into the signed
* range [1.0,2.0) or [-1.0,2.0).
*/
exp_a = spu_and((vec_ullong2)a, exp_mask);
exp_b = spu_and((vec_ullong2)b, exp_mask);
mant_a = spu_sel(a, one, (vec_ullong2)exp_mask);
mant_b = spu_sel(b, one, (vec_ullong2)exp_mask);
/* Approximate the single reciprocal of b by using
* the single precision reciprocal estimate followed by one
* single precision iteration of Newton-Raphson.
*/
mant_bf = spu_roundtf(mant_b);
inv_bf = spu_re(mant_bf);
inv_bf = spu_madd(spu_nmsub(mant_bf, inv_bf, onef), inv_bf, inv_bf);
/* Perform 2 more Newton-Raphson iterations in double precision. The
* result (q1) is in the range (0.5, 2.0).
*/
inv_b = spu_extend(inv_bf);
inv_b = spu_madd(spu_nmsub(mant_b, inv_b, one), inv_b, inv_b);
q0 = spu_mul(mant_a, inv_b);
q1 = spu_madd(spu_nmsub(mant_b, q0, mant_a), inv_b, q0);
/* Determine the exponent correction factor that must be applied
* to q1 by taking into account the exponent of the normalized inputs
* and the scale factors that were applied to normalize them.
*/
exp = spu_rlmaska(spu_sub((vec_int4)exp_a, (vec_int4)exp_b), -20);
exp = spu_add(exp, (vec_int4)spu_add(spu_and((vec_int4)a_denorm, -0x34), spu_and((vec_int4)b_denorm, 0x34)));
/* Bias the quotient exponent depending on the sign of the exponent correction
* factor so that a single multiplier will ensure the entire double precision
* domain (including denorms) can be achieved.
*
* exp bias q1 adjust exp
* ===== ======== ==========
* positive 2^+65 -65
* negative 2^-64 +64
*/
exp_bias = spu_xor(spu_rlmaska(exp, -31), 64);
exp = spu_sub(exp, exp_bias);
q1 = spu_sel(q1, (vec_double2)spu_add((vec_int4)q1, spu_sl(exp_bias, 20)), exp_mask);
/* Compute a multiplier (mult) to applied to the quotient (q1) to produce the
* expected result. On overflow, clamp the multiplier to the maximum non-infinite
* number in case the rounding mode is not round-to-nearest.
*/
exp = spu_add(exp, 0x3FF);
no_underflow = spu_cmpgt(exp, 0);
overflow = spu_cmpgt(exp, 0x7FE);
exp = spu_and(spu_sl(exp, 20), (vec_int4)no_underflow);
exp = spu_and(exp, (vec_int4)exp_mask);
mult = spu_sel((vec_double2)exp, (vec_double2)(spu_add((vec_uint4)exp_mask, -1)), (vec_ullong2)overflow);
/* Handle special value conditions. These include:
*
* 1) IF either operand is a NaN OR both operands are 0 or INFINITY THEN a NaN
* results.
* 2) ELSE IF the dividend is an INFINITY OR the divisor is 0 THEN a INFINITY results.
* 3) ELSE IF the dividend is 0 OR the divisor is INFINITY THEN a 0 results.
*/
mult = spu_andc(mult, (vec_double2)spu_or(a_zero, b_inf));
mult = spu_sel(mult, (vec_double2)exp_mask, spu_or(a_inf, b_zero));
nan = spu_or(a_nan, b_nan);
nan = spu_or(nan, spu_and(a_zero, b_zero));
nan = spu_or(nan, spu_and(a_inf, b_inf));
mult = spu_or(mult, (vec_double2)nan);
/* Scale the final quotient */
q2 = spu_mul(q1, mult);
return (q2);
}
static inline
vec_uint4 vec_Maj(vec_uint4 x, vec_uint4 y, vec_uint4 z)
{
return spu_sel(x, y, spu_xor(x, z));
}
void* libvector_pointwise_multiply_32fc_unaligned(void* target, void* src0, void* src1, unsigned int num_bytes){
//loop iterator i
int i = 0;
void* retval = target;
//put the target and source addresses into qwords
vector unsigned int address_counter_tgt = {(unsigned int)target, 0, 0, 0};
vector unsigned int address_counter_src0 = {(unsigned int)src0, 0, 0 ,0};
vector unsigned int address_counter_src1 = {(unsigned int)src1, 0, 0, 0};
//create shuffle masks
//shuffle mask building blocks:
//all from the first vector
vector unsigned char oneup = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
//all from the second vector
vector unsigned char second_oneup = {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f};
//gamma: second half of the second, first half of the first, break at (unsigned int)src0%16
vector unsigned char src_cmp = spu_splats((unsigned char)((unsigned int)src0%16));
vector unsigned char gt_res = spu_cmpgt(oneup, src_cmp);
vector unsigned char eq_res = spu_cmpeq(oneup, src_cmp);
vector unsigned char cmp_res = spu_or(gt_res, eq_res);
vector unsigned char sixteen_uchar = spu_splats((unsigned char)16);
vector unsigned char phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_gamma = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
shuffle_mask_gamma = spu_rlqwbyte(shuffle_mask_gamma, (unsigned int)src0%16);
//eta: second half of the second, first half of the first, break at (unsigned int)src1%16
src_cmp = spu_splats((unsigned char)((unsigned int)src1%16));
gt_res = spu_cmpgt(oneup, src_cmp);
eq_res = spu_cmpeq(oneup, src_cmp);
cmp_res = spu_or(gt_res, eq_res);
sixteen_uchar = spu_splats((unsigned char)16);
phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_eta = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
shuffle_mask_eta = spu_rlqwbyte(shuffle_mask_eta, (unsigned int)src1%16);
vector unsigned char tgt_second = spu_rlqwbyte(second_oneup, -((unsigned int)target%16));
vector unsigned char tgt_first = spu_rlqwbyte(oneup, -((unsigned int)target%16));
//alpha: first half of first, second half of second, break at (unsigned int)target%16
src_cmp = spu_splats((unsigned char)((unsigned int)target%16));
gt_res = spu_cmpgt(oneup, src_cmp);
eq_res = spu_cmpeq(oneup, src_cmp);
cmp_res = spu_or(gt_res, eq_res);
phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_alpha = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
//delta: first half of first, first half of second, break at (unsigned int)target%16
vector unsigned char shuffle_mask_delta = spu_shuffle(oneup, tgt_second, (vector unsigned char)shuffle_mask_alpha);
//epsilon: second half of second, second half of first, break at (unsigned int)target%16
vector unsigned char shuffle_mask_epsilon = spu_shuffle(tgt_second, oneup, (vector unsigned char)shuffle_mask_alpha);
//zeta: second half of second, first half of first, break at 16 - (unsigned int)target%16
vector unsigned int shuffle_mask_zeta = spu_rlqwbyte(shuffle_mask_alpha, (unsigned int)target%16);
//beta: first half of first, second half of second, break at num_bytes%16
src_cmp = spu_splats((unsigned char)(num_bytes%16));
gt_res = spu_cmpgt(oneup, src_cmp);
eq_res = spu_cmpeq(oneup, src_cmp);
cmp_res = spu_or(gt_res, eq_res);
phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_beta = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
qword src0_past;
qword src0_present;
qword src1_past;
qword src1_present;
qword tgt_past;
qword tgt_present;
qword in_temp0;
qword in_temp1;
qword out_temp0;
qword out_temp1;
src0_past = si_lqd((qword)address_counter_src0, 0);
src1_past = si_lqd((qword)address_counter_src1, 0);
tgt_past = si_lqd((qword)address_counter_tgt, 0);
vector unsigned char shuffle_mask_complexprod0 = {0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03,
0x0c, 0x0d, 0x0e, 0x0f, 0x08, 0x09, 0x0a, 0x0b};
vector unsigned char shuffle_mask_complexprod1 = {0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
0x08, 0x09, 0x0a, 0x0b, 0x18, 0x19, 0x1a, 0x1b};
vector unsigned char shuffle_mask_complexprod2 = {0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17,
0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f};
vector unsigned char sign_changer = {0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00};
vector float prod0;
qword shuf0;
vector float prod1;
vector float sign_change;
qword summand0;
qword summand1;
vector float sum;
for(i = 0; i < num_bytes/16; ++i) {
src0_present = si_lqd((qword)address_counter_src0, 16);
src1_present = si_lqd((qword)address_counter_src1, 16);
tgt_present = si_lqd((qword)address_counter_tgt, 16);
in_temp0 = spu_shuffle(src0_present, src0_past, (vector unsigned char)shuffle_mask_gamma);
in_temp1 = spu_shuffle(src1_present, src1_past, (vector unsigned char)shuffle_mask_eta);
prod0 = spu_mul((vector float)in_temp0, (vector float)in_temp1);
shuf0 = spu_shuffle((qword)in_temp1, (qword)in_temp1, shuffle_mask_complexprod0);
prod1 = spu_mul((vector float)in_temp0, (vector float)shuf0);
sign_change = spu_xor(prod0, (vector float)sign_changer);
summand0 = spu_shuffle((qword)sign_change, (qword)prod1, shuffle_mask_complexprod1);
summand1 = spu_shuffle((qword)sign_change, (qword)prod1, shuffle_mask_complexprod2);
sum = spu_add((vector float)summand0, (vector float)summand1);
out_temp0 = spu_shuffle(tgt_past, (qword)sum, shuffle_mask_delta);
out_temp1 = spu_shuffle(tgt_present, (qword)sum, shuffle_mask_epsilon);
si_stqd(out_temp0, (qword)address_counter_tgt, 0);
si_stqd(out_temp1, (qword)address_counter_tgt, 16);
tgt_past = out_temp1;
src0_past = src0_present;
src1_past = src1_present;
address_counter_src0 = spu_add(address_counter_src0, 16);
address_counter_src1 = spu_add(address_counter_src1, 16);
address_counter_tgt = spu_add(address_counter_tgt, 16);
}
src0_present = si_lqd((qword)address_counter_src0, 16);
src1_present = si_lqd((qword)address_counter_src1, 16);
tgt_present = si_lqd((qword)address_counter_tgt, 16);
in_temp0 = spu_shuffle(src0_present, src0_past, (vector unsigned char) shuffle_mask_gamma);
in_temp1 = spu_shuffle(src1_present, src1_past, (vector unsigned char) shuffle_mask_eta);
prod0 = spu_mul((vector float)in_temp0, (vector float)in_temp1);
shuf0 = spu_shuffle((qword)in_temp1, (qword)in_temp1, shuffle_mask_complexprod0);
prod1 = spu_mul(prod0, (vector float)shuf0);
sign_change = spu_xor(prod0, (vector float)sign_changer);
summand0 = spu_shuffle((qword)sign_change, (qword)prod1, shuffle_mask_complexprod1);
summand1 = spu_shuffle((qword)sign_change, (qword)prod1, shuffle_mask_complexprod2);
sum = spu_add((vector float)summand0, (vector float)summand1);
qword target_temp = spu_shuffle(tgt_present, tgt_past, (vector unsigned char) shuffle_mask_zeta);
qword meld = spu_shuffle((qword)sum, target_temp, (vector unsigned char)shuffle_mask_beta);
out_temp0 = spu_shuffle(tgt_past, meld, shuffle_mask_delta);
out_temp1 = spu_shuffle(tgt_present, meld, shuffle_mask_epsilon);
si_stqd(out_temp0, (qword)address_counter_tgt, 0);
si_stqd(out_temp1, (qword)address_counter_tgt, 16);
return retval;
}
