unsigned int
__mfc_tag_release (unsigned int tag)
{
vector unsigned int is_invalid;
vector unsigned int mask = (vector unsigned int)
{ 0x80000000, 0x80000000, 0x80000000, 0x80000000 };
vector signed int zero = (vector signed int) { 0, 0, 0, 0 };
vector signed int has_been_reserved;
/* Check if the tag is out of range. */
is_invalid = spu_cmpgt (spu_promote (tag, 0), 31);
/* Check whether the tag has been reserved, set to all 1 if has not
been reserved, 0 otherwise. */
has_been_reserved = (vector signed int) spu_rl (__mfc_tag_table, tag);
has_been_reserved = (vector signed int) spu_cmpgt (zero, has_been_reserved);
/* Set invalid. */
is_invalid = spu_or ((vector unsigned int) has_been_reserved, is_invalid);
mask = spu_rlmask (mask, (int)(-tag));
__mfc_tag_table = spu_or (__mfc_tag_table, mask);
return spu_extract(is_invalid, 0);
}
inline void merge_cache_blocks(RenderableCacheLine* cache)
{
vec_uchar16 next = cache->chunkNext;
for (;;) {
vec_uchar16 nextnext = spu_shuffle(next, next, next);
vec_uchar16 nextmask = spu_and(next, spu_splats((unsigned char)CHUNKNEXT_MASK));
vec_ushort8 firstblock0 = spu_cmpeq( cache->chunkStart[0], 0);
vec_ushort8 firstblock1 = spu_cmpeq( cache->chunkStart[1], 0);
// change next to word offset, note we don't care what the low bit shifted in is
vec_uchar16 firstshuf = (vec_uchar16) spu_sl( (vec_ushort8)nextmask, 1 );
vec_uchar16 first = (vec_uchar16) spu_shuffle( firstblock0, firstblock1, firstshuf );
vec_ushort8 tri0 = cache->chunkTriangle[0];
vec_ushort8 tri1 = cache->chunkTriangle[1];
vec_uchar16 trishufhi = spu_or ( firstshuf, spu_splats((unsigned char) 1));
vec_uchar16 trishuflo = spu_and( firstshuf, spu_splats((unsigned char) 254));
vec_ushort8 ntri0 = spu_shuffle( tri0, tri1, spu_shuffle( trishuflo, trishufhi, SHUF0 ) );
vec_ushort8 ntri1 = spu_shuffle( tri0, tri1, spu_shuffle( trishuflo, trishufhi, SHUF1 ) );
vec_ushort8 trieq0 = spu_cmpeq( tri0, ntri0 );
vec_ushort8 trieq1 = spu_cmpeq( tri1, ntri1 );
vec_uchar16 trieq = (vec_uchar16) spu_shuffle( trieq0, trieq1, MERGE );
vec_uchar16 combi = spu_orc(first, trieq);
vec_uchar16 canmerge = spu_cmpgt( spu_nor(spu_or(next, nextnext), combi), 256-CHUNKNEXT_BUSY_BIT );
vec_uint4 gather = spu_gather( canmerge );
vec_uint4 mergeid = spu_sub( spu_cntlz( gather ), spu_promote((unsigned int)16, 0));
if( !spu_extract(gather, 0) ) {
return;
}
// unsigned int firstchunk = spu_extract(mergeid, 0);
// unsigned int nextchunk = cache->chunkNextArray[firstchunk];
vec_uint4 v_chunkNext = (vec_uint4) si_rotqby( (qword) next, (qword) spu_add(mergeid,13) );
vec_uint4 v_chunkNextNext = (vec_uint4) si_rotqby( (qword) next, (qword) spu_add(v_chunkNext,13) );
// cache->chunkNextArray[firstchunk] = cache->chunkNextArray[nextchunk];
next = spu_shuffle( (vec_uchar16) v_chunkNextNext, next, (vec_uchar16) si_cbd( (qword) mergeid, 0 ) );
// cache->chunkNextArray[nextchunk] = CHUNKNEXT_FREE_BLOCK;
next = spu_shuffle( spu_splats( (unsigned char) CHUNKNEXT_FREE_BLOCK), next, (vec_uchar16) si_cbd( (qword) v_chunkNext, 0 ) );
// this is for debug use only, it's not really needed...
// cache->chunkStartArray[nextchunk] = -1;
cache->chunkStartArray[ spu_extract(v_chunkNext,0) & 255 ] = -1;
cache->chunkNext = next;
}
}
/* returns rough sin(x) approximation. the period is 2 (real sin has 2*Pi period). */
inline vec_float4 fast_sinf(vec_float4 x) {
vec_uint4 xi = (vec_uint4)x;
xi = spu_and(xi, 0x7fffffffU);
vec_uint4 ui = spu_convtu((vec_float4)xi, 0);
vec_float4 neg = (vec_float4)spu_sl(ui, 31u);
vec_float4 t = spu_splats(.5f) - (((vec_float4)xi) - spu_convtf(ui, 0));
vec_float4 s = spu_splats(1.f) - spu_splats(4.f)*t*t;
return spu_or(s, neg);
}
unsigned int
__mfc_multi_tag_release (unsigned int first_tag, unsigned int number_of_tags)
{
vector unsigned int table_copy, tmp, tmp1;
vector unsigned int one = (vector unsigned int)
{ 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF };
vector unsigned int is_invalid;
unsigned int last_tag;
vector unsigned int has_been_reserved;
last_tag = first_tag + number_of_tags;
table_copy = spu_sl (one, number_of_tags);
table_copy = spu_rl (table_copy, -last_tag);
table_copy = spu_xor (table_copy, -1);
/* Make sure the tags are in range and valid. */
tmp = spu_cmpgt (spu_promote(last_tag, 0), 32);
tmp1 = spu_cmpgt (spu_promote(number_of_tags, 0), 32);
is_invalid = spu_cmpgt (spu_promote(first_tag, 0), 31);
/* All bits are set to 1 if invalid, 0 if valid. */
is_invalid = spu_or (tmp, is_invalid);
is_invalid = spu_or (tmp1, is_invalid);
/* check whether these tags have been reserved */
tmp = spu_rlmask (one, (int)-number_of_tags);
tmp1 = spu_sl (__mfc_tag_table, first_tag);
has_been_reserved = spu_cmpgt(tmp1, tmp);
is_invalid = spu_or (has_been_reserved, is_invalid);
table_copy = spu_sel (__mfc_tag_table, table_copy, table_copy);
__mfc_tag_table = spu_sel (table_copy, __mfc_tag_table, is_invalid);
return spu_extract (is_invalid, 0);
}
vector double
__divv2df3 (vector double a_in, vector double b_in)
{
/* Variables */
vec_int4 exp, exp_bias;
vec_uint4 no_underflow, overflow;
vec_float4 mant_bf, inv_bf;
vec_ullong2 exp_a, exp_b;
vec_ullong2 a_nan, a_zero, a_inf, a_denorm, a_denorm0;
vec_ullong2 b_nan, b_zero, b_inf, b_denorm, b_denorm0;
vec_ullong2 nan;
vec_uint4 a_exp, b_exp;
vec_ullong2 a_mant_0, b_mant_0;
vec_ullong2 a_exp_1s, b_exp_1s;
vec_ullong2 sign_exp_mask;
vec_double2 a, b;
vec_double2 mant_a, mant_b, inv_b, q0, q1, q2, mult;
/* Constants */
vec_uint4 exp_mask_u32 = spu_splats((unsigned int)0x7FF00000);
vec_uchar16 splat_hi = (vec_uchar16) {
0,1,2,3, 0,1,2,3, 8, 9,10,11, 8,9,10,11
};
vec_uchar16 swap_32 = (vec_uchar16) {
4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11
};
vec_ullong2 exp_mask = spu_splats(0x7FF0000000000000ULL);
vec_ullong2 sign_mask = spu_splats(0x8000000000000000ULL);
vec_float4 onef = spu_splats(1.0f);
vec_double2 one = spu_splats(1.0);
vec_double2 exp_53 = (vec_double2)spu_splats(0x0350000000000000ULL);
sign_exp_mask = spu_or(sign_mask, exp_mask);
/* Extract the floating point components from each of the operands including
* exponent and mantissa.
*/
a_exp = (vec_uint4)spu_and((vec_uint4)a_in, exp_mask_u32);
a_exp = spu_shuffle(a_exp, a_exp, splat_hi);
b_exp = (vec_uint4)spu_and((vec_uint4)b_in, exp_mask_u32);
b_exp = spu_shuffle(b_exp, b_exp, splat_hi);
a_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)a_in, sign_exp_mask), 0);
a_mant_0 = spu_and(a_mant_0, spu_shuffle(a_mant_0, a_mant_0, swap_32));
b_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)b_in, sign_exp_mask), 0);
b_mant_0 = spu_and(b_mant_0, spu_shuffle(b_mant_0, b_mant_0, swap_32));
a_exp_1s = (vec_ullong2)spu_cmpeq(a_exp, exp_mask_u32);
b_exp_1s = (vec_ullong2)spu_cmpeq(b_exp, exp_mask_u32);
/* Identify all possible special values that must be accommodated including:
* +-denorm, +-0, +-infinity, and NaNs.
*/
a_denorm0= (vec_ullong2)spu_cmpeq(a_exp, 0);
a_nan = spu_andc(a_exp_1s, a_mant_0);
a_zero = spu_and (a_denorm0, a_mant_0);
a_inf = spu_and (a_exp_1s, a_mant_0);
a_denorm = spu_andc(a_denorm0, a_zero);
b_denorm0= (vec_ullong2)spu_cmpeq(b_exp, 0);
b_nan = spu_andc(b_exp_1s, b_mant_0);
b_zero = spu_and (b_denorm0, b_mant_0);
b_inf = spu_and (b_exp_1s, b_mant_0);
b_denorm = spu_andc(b_denorm0, b_zero);
/* Scale denorm inputs to into normalized numbers by conditionally scaling the
* input parameters.
*/
a = spu_sub(spu_or(a_in, exp_53), spu_sel(exp_53, a_in, sign_mask));
a = spu_sel(a_in, a, a_denorm);
b = spu_sub(spu_or(b_in, exp_53), spu_sel(exp_53, b_in, sign_mask));
b = spu_sel(b_in, b, b_denorm);
/* Extract the divisor and dividend exponent and force parameters into the signed
* range [1.0,2.0) or [-1.0,2.0).
*/
exp_a = spu_and((vec_ullong2)a, exp_mask);
exp_b = spu_and((vec_ullong2)b, exp_mask);
mant_a = spu_sel(a, one, (vec_ullong2)exp_mask);
mant_b = spu_sel(b, one, (vec_ullong2)exp_mask);
/* Approximate the single reciprocal of b by using
* the single precision reciprocal estimate followed by one
* single precision iteration of Newton-Raphson.
*/
mant_bf = spu_roundtf(mant_b);
inv_bf = spu_re(mant_bf);
inv_bf = spu_madd(spu_nmsub(mant_bf, inv_bf, onef), inv_bf, inv_bf);
/* Perform 2 more Newton-Raphson iterations in double precision. The
* result (q1) is in the range (0.5, 2.0).
*/
inv_b = spu_extend(inv_bf);
inv_b = spu_madd(spu_nmsub(mant_b, inv_b, one), inv_b, inv_b);
q0 = spu_mul(mant_a, inv_b);
q1 = spu_madd(spu_nmsub(mant_b, q0, mant_a), inv_b, q0);
/* Determine the exponent correction factor that must be applied
* to q1 by taking into account the exponent of the normalized inputs
* and the scale factors that were applied to normalize them.
*/
exp = spu_rlmaska(spu_sub((vec_int4)exp_a, (vec_int4)exp_b), -20);
exp = spu_add(exp, (vec_int4)spu_add(spu_and((vec_int4)a_denorm, -0x34), spu_and((vec_int4)b_denorm, 0x34)));
/* Bias the quotient exponent depending on the sign of the exponent correction
* factor so that a single multiplier will ensure the entire double precision
* domain (including denorms) can be achieved.
*
* exp bias q1 adjust exp
* ===== ======== ==========
* positive 2^+65 -65
* negative 2^-64 +64
*/
exp_bias = spu_xor(spu_rlmaska(exp, -31), 64);
exp = spu_sub(exp, exp_bias);
q1 = spu_sel(q1, (vec_double2)spu_add((vec_int4)q1, spu_sl(exp_bias, 20)), exp_mask);
/* Compute a multiplier (mult) to applied to the quotient (q1) to produce the
* expected result. On overflow, clamp the multiplier to the maximum non-infinite
* number in case the rounding mode is not round-to-nearest.
*/
exp = spu_add(exp, 0x3FF);
no_underflow = spu_cmpgt(exp, 0);
overflow = spu_cmpgt(exp, 0x7FE);
exp = spu_and(spu_sl(exp, 20), (vec_int4)no_underflow);
exp = spu_and(exp, (vec_int4)exp_mask);
mult = spu_sel((vec_double2)exp, (vec_double2)(spu_add((vec_uint4)exp_mask, -1)), (vec_ullong2)overflow);
/* Handle special value conditions. These include:
*
* 1) IF either operand is a NaN OR both operands are 0 or INFINITY THEN a NaN
* results.
* 2) ELSE IF the dividend is an INFINITY OR the divisor is 0 THEN a INFINITY results.
* 3) ELSE IF the dividend is 0 OR the divisor is INFINITY THEN a 0 results.
*/
mult = spu_andc(mult, (vec_double2)spu_or(a_zero, b_inf));
mult = spu_sel(mult, (vec_double2)exp_mask, spu_or(a_inf, b_zero));
nan = spu_or(a_nan, b_nan);
nan = spu_or(nan, spu_and(a_zero, b_zero));
nan = spu_or(nan, spu_and(a_inf, b_inf));
mult = spu_or(mult, (vec_double2)nan);
/* Scale the final quotient */
q2 = spu_mul(q1, mult);
return (q2);
}
void process_render_tasks(unsigned long eah_render_tasks, unsigned long eal_render_tasks)
{
const vec_uchar16 SHUFFLE_MERGE_BYTES = (vec_uchar16) { // merge lo bytes from unsigned shorts (array)
1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31
};
const vec_uchar16 SHUFFLE_GET_BUSY_WITH_ONES = (vec_uchar16) { // get busy flag with ones in unused bytes
0xc0, 0xc0, 2, 3, 0xc0,0xc0,0xc0,0xc0, 0xc0,0xc0,0xc0,0xc0
};
const vec_uchar16 ZERO_BYTES = (vec_uchar16) spu_splats(0);
char trianglebuffer[ 256 + TRIANGLE_MAX_SIZE ];
char sync_buffer[128+127];
void* aligned_sync_buffer = (void*) ( ((unsigned long)sync_buffer+127) & ~127 );
RenderableCacheLine* cache = (RenderableCacheLine*) aligned_sync_buffer;
unsigned long long cache_ea;
spu_mfcdma64(&cache_ea, eah_render_tasks, eal_render_tasks, sizeof(cache_ea), 0, MFC_GET_CMD);
mfc_write_tag_mask(1<<0);
mfc_read_tag_status_all();
while (cache_ea) {
// terminate immediately if possible
if (spu_stat_in_mbox())
return;
// read the cache line
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_GETLLAR_CMD);
spu_readch(MFC_RdAtomicStat);
unsigned int endTriangle = cache->endTriangle;
vec_ushort8 testTriangle = spu_splats((unsigned short) endTriangle);
// first look for short chunks
vec_uchar16 next = cache->chunkNext;
vec_uchar16 nextmask = spu_and(next, spu_splats((unsigned char)CHUNKNEXT_MASK));
// change next to word offset, note we don't care what the low bit shifted in is
vec_uchar16 firstshuf = (vec_uchar16) spu_sl( (vec_ushort8)nextmask, 1 );
vec_uchar16 trishufhi = spu_or ( firstshuf, spu_splats((unsigned char) 1));
vec_uchar16 trishuflo = spu_and( firstshuf, spu_splats((unsigned char) 254));
vec_ushort8 start0 = cache->chunkStart[0];
vec_ushort8 start1 = cache->chunkStart[1];
vec_ushort8 nstart0 = spu_shuffle( start0, start1, spu_shuffle( trishuflo, trishufhi, SHUF0 ) );
vec_ushort8 nstart1 = spu_shuffle( start0, start1, spu_shuffle( trishuflo, trishufhi, SHUF1 ) );
vec_ushort8 starteq0 = spu_cmpeq( nstart0, spu_splats((unsigned short)0) );
vec_ushort8 starteq1 = spu_cmpeq( nstart1, spu_splats((unsigned short)0) );
vec_ushort8 end0 = spu_sel( nstart0, spu_splats((unsigned short)4096), starteq0);
vec_ushort8 end1 = spu_sel( nstart1, spu_splats((unsigned short)4096), starteq1);
vec_ushort8 len0 = spu_sub( end0, start0);
vec_ushort8 len1 = spu_sub( end1, start1);
vec_ushort8 small0 = spu_cmpgt( spu_splats((unsigned short)17), len0);
vec_ushort8 small1 = spu_cmpgt( spu_splats((unsigned short)17), len1);
vec_uchar16 small = (vec_uchar16) spu_shuffle( small0, small1, MERGE );
vec_uint4 smallChunkGather = spu_gather(small);
// check to see if chunk is already at the last triangle
vec_uint4 doneChunkGather = spu_gather( (vec_uchar16) spu_shuffle(
(vec_uchar16) spu_cmpeq(testTriangle, cache->chunkTriangle[0]),
(vec_uchar16) spu_cmpeq(testTriangle, cache->chunkTriangle[1]),
SHUFFLE_MERGE_BYTES) );
// check if the chunk is free
vec_uint4 freeChunkGather = spu_gather(
spu_cmpeq( spu_splats( (unsigned char) CHUNKNEXT_FREE_BLOCK ), cache->chunkNext ) );
// check to see if the chunk is being processed
vec_uint4 busyChunkGather = spu_gather(
spu_cmpgt( cache->chunkNext, //spu_and(cache->chunkNext, CHUNKNEXT_MASK),
spu_splats( (unsigned char) (CHUNKNEXT_BUSY_BIT-1) ) ) );
// doneChunkGather, freeChunkGather, busyChunkGather - rightmost 16 bits of word 0
// note that if freeChunkGather is true then busyChunkGather must also be true
// done=false, free=false, busy=false -> can process
// free=false, busy=false -> can be merged
// decide which chunk to process
vec_uint4 mayProcessGather = spu_nor( doneChunkGather, busyChunkGather );
vec_uint4 mayProcessShortGather = spu_and( mayProcessGather, smallChunkGather );
vec_uint4 shortSelMask = spu_cmpeq( mayProcessShortGather, spu_splats(0U) );
vec_uint4 mayProcessSelection = spu_sel( mayProcessShortGather, mayProcessGather, shortSelMask );
/*
if (!spu_extract(shortSelMask, 0))
printf("taken short: may=%04x short=%04x mayshort=%04x mask=%04x sel=%04x\n",
spu_extract(mayProcessGather, 0) & 0xffff,
spu_extract(smallChunkGather, 0),
spu_extract(mayProcessShortGather, 0),
spu_extract(shortSelMask, 0) & 0xffff,
spu_extract(mayProcessSelection, 0) & 0xffff );
*/
vec_uint4 mayProcessBits = spu_sl( mayProcessSelection, 16);
unsigned int chunkToProcess = spu_extract( spu_cntlz( mayProcessBits ), 0);
unsigned int freeChunk = spu_extract( spu_cntlz( spu_sl( freeChunkGather, 16 ) ), 0);
// if there's nothing to process, try the next cache line in the rendering tasks list
if (!spu_extract(mayProcessBits, 0)) {
trynextcacheline:
cache_ea = cache->next;
// sleep();
continue;
}
unsigned int chunkStart = cache->chunkStartArray [chunkToProcess];
unsigned int chunkTriangle = cache->chunkTriangleArray[chunkToProcess];
unsigned int chunkNext = cache->chunkNextArray [chunkToProcess] & CHUNKNEXT_MASK;
unsigned int chunkEnd = (cache->chunkStartArray [chunkNext]-1) & (NUMBER_OF_TILES-1);
unsigned int chunkLength = 1 + chunkEnd-chunkStart;
// only need an extra block if the block is especially long
if (chunkLength <= NUMBER_OF_TILES_PER_CHUNK) {
freeChunk = 32;
}
// mark this block as busy
cache->chunkNextArray[chunkToProcess] |= CHUNKNEXT_BUSY_BIT;
// if there's at least one free chunk, claim it
if (freeChunk != 32) {
cache->chunkNextArray[freeChunk] = CHUNKNEXT_RESERVED;
cache->chunkTriangleArray[freeChunk] = chunkTriangle;
}
// write the cache line back
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_PUTLLC_CMD);
if (spu_readch(MFC_RdAtomicStat) & MFC_PUTLLC_STATUS)
continue;
#ifdef INFO
printf("[%d] Claimed chunk %d (%d-%d len %d) at tri %x end %x with free chunk %d\n", _SPUID,
chunkToProcess, chunkStart, chunkEnd, chunkLength, chunkTriangle, endTriangle,
freeChunk!=32 ? freeChunk : -1 );
// debug_render_tasks(cache);
#endif
Triangle* triangle;
int firstTile;
do {
// read the triangle data for the current triangle
unsigned int extra = chunkTriangle & 127;
unsigned long long trianglebuffer_ea = cache_ea + TRIANGLE_OFFSET_FROM_CACHE_LINE + (chunkTriangle & ~127);
triangle = (Triangle*) (trianglebuffer+extra);
unsigned int length = (extra + TRIANGLE_MAX_SIZE + 127) & ~127;
// ensure DMA slot available
do {} while (!spu_readchcnt(MFC_Cmd));
spu_mfcdma64(trianglebuffer, mfc_ea2h(trianglebuffer_ea), mfc_ea2l(trianglebuffer_ea),
length, 0, MFC_GET_CMD);
mfc_write_tag_mask(1<<0);
mfc_read_tag_status_all();
// get the triangle deltas
firstTile = findFirstTriangleTile(triangle, chunkStart, chunkEnd);
if (firstTile>=0)
break;
// no match, try next triangle
chunkTriangle = triangle->next_triangle;
} while (chunkTriangle != endTriangle);
// if we actually have something to process...
if (firstTile>=0) {
// the "normal" splitting will now become:
// chunkStart .. (firstTile-1) -> triangle->next_triangle
// firstTile .. (firstTile+NUM-1) -> chunkTriangle (BUSY)
// (firstTile+NUM) .. chunkEnd -> chunkTriangle (FREE)
int tailChunk;
int thisChunk;
int nextBlockStart;
int thisBlockStart;
int realBlockStart;
do {
retry:
// read the cache line
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_GETLLAR_CMD);
spu_readch(MFC_RdAtomicStat);
// calculate start of next block
nextBlockStart = firstTile + NUMBER_OF_TILES_PER_CHUNK;
if (nextBlockStart > chunkEnd)
nextBlockStart = chunkEnd+1;
// calculate start of block to mark as busy
thisBlockStart = nextBlockStart - NUMBER_OF_TILES_PER_CHUNK;
if (thisBlockStart < chunkStart)
thisBlockStart = chunkStart;
realBlockStart = thisBlockStart;
#ifdef INFO
printf("[%d] nextBlockStart=%d, realBlockStart=%d, thisBlockStart=%d, chunkStart=%d\n", _SPUID,
nextBlockStart, realBlockStart, thisBlockStart, chunkStart);
#endif
// allocate some more free chunks
vec_uint4 freeChunkGather2 = spu_sl(spu_gather(spu_cmpeq(
spu_splats((unsigned char)CHUNKNEXT_FREE_BLOCK), cache->chunkNext)), 16);
unsigned int freeChunk2 = spu_extract(spu_cntlz(freeChunkGather2), 0);
if (freeChunk == 32) {
// if we didn't have one before, try again
freeChunk = freeChunk2;
// and try to get the second one
freeChunkGather2 = spu_andc( freeChunkGather2, spu_promote(0x80000000>>freeChunk2, 0) );
freeChunk2 = spu_extract(spu_cntlz(freeChunkGather2), 0);
} else {
// speculatively clear the free chunk just in case we don't need it
cache->chunkNextArray[freeChunk] = CHUNKNEXT_FREE_BLOCK;
}
#ifdef INFO
printf("[%d] Free chunks %d and %d, cN=%d, nBS=%d, cE=%d, tBS=%d, cS=%d\n",
_SPUID, freeChunk, freeChunk2, chunkNext, nextBlockStart, chunkEnd, thisBlockStart, chunkStart );
#endif
// mark region after as available for processing if required
if (nextBlockStart < chunkEnd) {
if (freeChunk==32) {
// if no free chunk, relinquish entire block and write back
cache->chunkNextArray[chunkToProcess] = chunkNext;
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_PUTLLC_CMD);
// if writeback failed, we *might* have a free block, retry
if (spu_readch(MFC_RdAtomicStat) & MFC_PUTLLC_STATUS)
goto retry;
// otherwise give up and try the next cache line
goto trynextcacheline;
}
cache->chunkStartArray[freeChunk] = nextBlockStart;
cache->chunkNextArray[freeChunk] = chunkNext;
cache->chunkTriangleArray[freeChunk] = chunkTriangle;
cache->chunkNextArray[chunkToProcess] = freeChunk | CHUNKNEXT_BUSY_BIT;
tailChunk = freeChunk;
#ifdef INFO
printf("[%d] Insert tail, tailChunk=%d, chunkNext=%d, chunkToProcess=%d\n", _SPUID, tailChunk, chunkNext, chunkToProcess);
debug_render_tasks(cache);
#endif
} else {
// we're gonna use freeChunk2 for the "in front" block, as we've not
// used freeChunk, let's use it as it's more likely to have a free chunk
freeChunk2 = freeChunk;
tailChunk = chunkNext;
}
// mark region before as available if required and possible
thisChunk = chunkToProcess;
if (thisBlockStart > chunkStart) {
if (freeChunk2 != 32) {
// mark this region as busy
cache->chunkStartArray[freeChunk2]=thisBlockStart;
cache->chunkNextArray[freeChunk2]=tailChunk | CHUNKNEXT_BUSY_BIT;
cache->chunkTriangleArray[freeChunk2]=chunkTriangle;
// mark region before as available for processing
cache->chunkNextArray[chunkToProcess]=freeChunk2;
cache->chunkTriangleArray[chunkToProcess]=triangle->next_triangle;
thisChunk = freeChunk2;
#ifdef INFO
printf("[%d] Insert new head, tailChunk=%d, chunkNext=%d, thisChunk=%d\n", _SPUID, tailChunk, chunkNext, thisChunk);
debug_render_tasks(cache);
#endif
} else {
// need to keep whole block, update info and mark bust
cache->chunkTriangleArray[chunkToProcess]=chunkTriangle;
cache->chunkNextArray[chunkToProcess]=tailChunk | CHUNKNEXT_BUSY_BIT;
realBlockStart = chunkStart;
printf("[%d] Keep whole block, tailChunk=%d, chunkNext=%d, thisChunk=%d\n", _SPUID, tailChunk, chunkNext, thisChunk);
debug_render_tasks(cache);
#ifdef INFO
#endif
sleep();
}
}
// merge chunks
merge_cache_blocks(cache);
// write the cache line back
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_PUTLLC_CMD);
} while (spu_readch(MFC_RdAtomicStat) & MFC_PUTLLC_STATUS);
// finally after the write succeeded, update the variables
chunkNext = tailChunk;
chunkToProcess = thisChunk;
chunkStart = firstTile; //thisBlockStart;
chunkLength = nextBlockStart - firstTile;
chunkEnd = chunkStart + chunkLength - 1;
freeChunk = 32;
// now we can process the block up to endTriangle
initTileBuffers(thisBlockStart, chunkEnd);
int ok=0;
while (chunkTriangle != endTriangle) {
#ifdef INFO
printf("[%d] Processing chunk %d at %4d len %4d, triangle %04x first=%d tbs=%d\n",
_SPUID, chunkToProcess, chunkStart, chunkLength,
chunkTriangle, firstTile, thisBlockStart);
#endif
// and actually process that triangle on these chunks
processTriangleChunks(triangle, cache, thisBlockStart, chunkEnd, chunkTriangle, ok);
ok=1;
#ifdef PAUSE
sleep();
#endif
// and advance to the next-triangle
chunkTriangle = triangle->next_triangle;
// this should only ever happen if we're running really low on cache line slots
// basically, if we pick up a block with more than NUMBER_OF_TILES_PER_CHUNK and
// there's no slot to store the pre-NUMBER_OF_TILES_PER_CHUNK tiles.
// in this case, we process from thisBlockStart only (because we know that from
// chunkStart to there has no result) and then we only process one triangle
if (chunkStart != realBlockStart) {
/*
printf("[%d] chunkStart=%d != realBlockStart %d, chunkEnd=%d, "
"firstTile=%d chunk=%d\n",
_SPUID, chunkStart, realBlockStart, chunkEnd,
firstTile, chunkToProcess);
debug_render_tasks(cache);
*/
// abort the while loop
break;
}
// read the next triangle
unsigned int extra = chunkTriangle & 127;
unsigned long long trianglebuffer_ea = cache_ea + TRIANGLE_OFFSET_FROM_CACHE_LINE + (chunkTriangle & ~127);
triangle = (Triangle*) (trianglebuffer+extra);
unsigned int length = (extra + TRIANGLE_MAX_SIZE + 127) & ~127;
// ensure DMA slot available
do {} while (!spu_readchcnt(MFC_Cmd));
spu_mfcdma64(trianglebuffer, mfc_ea2h(trianglebuffer_ea),
mfc_ea2l(trianglebuffer_ea), length, 0, MFC_GET_CMD);
mfc_write_tag_mask(1<<0);
mfc_read_tag_status_all();
} // until chunkTriangle == endTriangle
// flush any output buffers
flushTileBuffers(thisBlockStart, chunkEnd);
} // firstTile>=0
static inline
vec_uint4 vec_Maj(vec_uint4 x, vec_uint4 y, vec_uint4 z)
{
return spu_sel(spu_and(y, z), spu_or(y, z), x);
}
void* libvector_pointwise_multiply_32fc_unaligned(void* target, void* src0, void* src1, unsigned int num_bytes){
//loop iterator i
int i = 0;
void* retval = target;
//put the target and source addresses into qwords
vector unsigned int address_counter_tgt = {(unsigned int)target, 0, 0, 0};
vector unsigned int address_counter_src0 = {(unsigned int)src0, 0, 0 ,0};
vector unsigned int address_counter_src1 = {(unsigned int)src1, 0, 0, 0};
//create shuffle masks
//shuffle mask building blocks:
//all from the first vector
vector unsigned char oneup = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
//all from the second vector
vector unsigned char second_oneup = {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f};
//gamma: second half of the second, first half of the first, break at (unsigned int)src0%16
vector unsigned char src_cmp = spu_splats((unsigned char)((unsigned int)src0%16));
vector unsigned char gt_res = spu_cmpgt(oneup, src_cmp);
vector unsigned char eq_res = spu_cmpeq(oneup, src_cmp);
vector unsigned char cmp_res = spu_or(gt_res, eq_res);
vector unsigned char sixteen_uchar = spu_splats((unsigned char)16);
vector unsigned char phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_gamma = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
shuffle_mask_gamma = spu_rlqwbyte(shuffle_mask_gamma, (unsigned int)src0%16);
//eta: second half of the second, first half of the first, break at (unsigned int)src1%16
src_cmp = spu_splats((unsigned char)((unsigned int)src1%16));
gt_res = spu_cmpgt(oneup, src_cmp);
eq_res = spu_cmpeq(oneup, src_cmp);
cmp_res = spu_or(gt_res, eq_res);
sixteen_uchar = spu_splats((unsigned char)16);
phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_eta = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
shuffle_mask_eta = spu_rlqwbyte(shuffle_mask_eta, (unsigned int)src1%16);
vector unsigned char tgt_second = spu_rlqwbyte(second_oneup, -((unsigned int)target%16));
vector unsigned char tgt_first = spu_rlqwbyte(oneup, -((unsigned int)target%16));
//alpha: first half of first, second half of second, break at (unsigned int)target%16
src_cmp = spu_splats((unsigned char)((unsigned int)target%16));
gt_res = spu_cmpgt(oneup, src_cmp);
eq_res = spu_cmpeq(oneup, src_cmp);
cmp_res = spu_or(gt_res, eq_res);
phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_alpha = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
//delta: first half of first, first half of second, break at (unsigned int)target%16
vector unsigned char shuffle_mask_delta = spu_shuffle(oneup, tgt_second, (vector unsigned char)shuffle_mask_alpha);
//epsilon: second half of second, second half of first, break at (unsigned int)target%16
vector unsigned char shuffle_mask_epsilon = spu_shuffle(tgt_second, oneup, (vector unsigned char)shuffle_mask_alpha);
//zeta: second half of second, first half of first, break at 16 - (unsigned int)target%16
vector unsigned int shuffle_mask_zeta = spu_rlqwbyte(shuffle_mask_alpha, (unsigned int)target%16);
//beta: first half of first, second half of second, break at num_bytes%16
src_cmp = spu_splats((unsigned char)(num_bytes%16));
gt_res = spu_cmpgt(oneup, src_cmp);
eq_res = spu_cmpeq(oneup, src_cmp);
cmp_res = spu_or(gt_res, eq_res);
phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_beta = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
qword src0_past;
qword src0_present;
qword src1_past;
qword src1_present;
qword tgt_past;
qword tgt_present;
qword in_temp0;
qword in_temp1;
qword out_temp0;
qword out_temp1;
src0_past = si_lqd((qword)address_counter_src0, 0);
src1_past = si_lqd((qword)address_counter_src1, 0);
tgt_past = si_lqd((qword)address_counter_tgt, 0);
vector unsigned char shuffle_mask_complexprod0 = {0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03,
0x0c, 0x0d, 0x0e, 0x0f, 0x08, 0x09, 0x0a, 0x0b};
vector unsigned char shuffle_mask_complexprod1 = {0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
0x08, 0x09, 0x0a, 0x0b, 0x18, 0x19, 0x1a, 0x1b};
vector unsigned char shuffle_mask_complexprod2 = {0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17,
0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f};
vector unsigned char sign_changer = {0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00};
vector float prod0;
qword shuf0;
vector float prod1;
vector float sign_change;
qword summand0;
qword summand1;
vector float sum;
for(i = 0; i < num_bytes/16; ++i) {
src0_present = si_lqd((qword)address_counter_src0, 16);
src1_present = si_lqd((qword)address_counter_src1, 16);
tgt_present = si_lqd((qword)address_counter_tgt, 16);
in_temp0 = spu_shuffle(src0_present, src0_past, (vector unsigned char)shuffle_mask_gamma);
in_temp1 = spu_shuffle(src1_present, src1_past, (vector unsigned char)shuffle_mask_eta);
prod0 = spu_mul((vector float)in_temp0, (vector float)in_temp1);
shuf0 = spu_shuffle((qword)in_temp1, (qword)in_temp1, shuffle_mask_complexprod0);
prod1 = spu_mul((vector float)in_temp0, (vector float)shuf0);
sign_change = spu_xor(prod0, (vector float)sign_changer);
summand0 = spu_shuffle((qword)sign_change, (qword)prod1, shuffle_mask_complexprod1);
summand1 = spu_shuffle((qword)sign_change, (qword)prod1, shuffle_mask_complexprod2);
sum = spu_add((vector float)summand0, (vector float)summand1);
out_temp0 = spu_shuffle(tgt_past, (qword)sum, shuffle_mask_delta);
out_temp1 = spu_shuffle(tgt_present, (qword)sum, shuffle_mask_epsilon);
si_stqd(out_temp0, (qword)address_counter_tgt, 0);
si_stqd(out_temp1, (qword)address_counter_tgt, 16);
tgt_past = out_temp1;
src0_past = src0_present;
src1_past = src1_present;
address_counter_src0 = spu_add(address_counter_src0, 16);
address_counter_src1 = spu_add(address_counter_src1, 16);
address_counter_tgt = spu_add(address_counter_tgt, 16);
}
src0_present = si_lqd((qword)address_counter_src0, 16);
src1_present = si_lqd((qword)address_counter_src1, 16);
tgt_present = si_lqd((qword)address_counter_tgt, 16);
in_temp0 = spu_shuffle(src0_present, src0_past, (vector unsigned char) shuffle_mask_gamma);
in_temp1 = spu_shuffle(src1_present, src1_past, (vector unsigned char) shuffle_mask_eta);
prod0 = spu_mul((vector float)in_temp0, (vector float)in_temp1);
shuf0 = spu_shuffle((qword)in_temp1, (qword)in_temp1, shuffle_mask_complexprod0);
prod1 = spu_mul(prod0, (vector float)shuf0);
sign_change = spu_xor(prod0, (vector float)sign_changer);
summand0 = spu_shuffle((qword)sign_change, (qword)prod1, shuffle_mask_complexprod1);
summand1 = spu_shuffle((qword)sign_change, (qword)prod1, shuffle_mask_complexprod2);
sum = spu_add((vector float)summand0, (vector float)summand1);
qword target_temp = spu_shuffle(tgt_present, tgt_past, (vector unsigned char) shuffle_mask_zeta);
qword meld = spu_shuffle((qword)sum, target_temp, (vector unsigned char)shuffle_mask_beta);
out_temp0 = spu_shuffle(tgt_past, meld, shuffle_mask_delta);
out_temp1 = spu_shuffle(tgt_present, meld, shuffle_mask_epsilon);
si_stqd(out_temp0, (qword)address_counter_tgt, 0);
si_stqd(out_temp1, (qword)address_counter_tgt, 16);
return retval;
}
