unsigned int __mfc_multi_tag_reserve (unsigned int number_of_tags) { vector unsigned int table_copy; vector unsigned int one = (vector unsigned int) { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; vector unsigned int count_busy, is_valid; vector unsigned int count_total; vector unsigned int count_avail = (vector unsigned int) { 0, 0, 0, 0 }; vector unsigned int index = (vector unsigned int) { 0, 0, 0, 0 }; table_copy = __mfc_tag_table; /* count_busy: number of consecutive busy tags count_avail: number of consecutive free tags table_copy: temporary copy of the tag table count_total: sum of count_busy and count_avail index: index of the current working tag */ do { table_copy = spu_sl (table_copy, count_avail); count_busy = spu_cntlz (table_copy); table_copy = spu_sl (table_copy, count_busy); count_avail = spu_cntlz (spu_xor(table_copy, -1)); count_total = spu_add (count_busy, count_avail); index = spu_add (index, count_total); } while (spu_extract (count_avail, 0) < number_of_tags && spu_extract (table_copy, 0) != 0); index = spu_sub (index, count_avail); /* is_valid is set to 0xFFFFFFFF if table_copy == 0, 0 otherwise. */ is_valid = spu_cmpeq (table_copy, 0); index = spu_sel (index, is_valid, is_valid); /* Now I need to actually mark the tags as used. */ table_copy = spu_sl (one, number_of_tags); table_copy = spu_rl (table_copy, -number_of_tags - spu_extract (index, 0)); table_copy = spu_sel (table_copy, __mfc_tag_table, table_copy); __mfc_tag_table = spu_sel (table_copy, __mfc_tag_table, is_valid); return spu_extract (index, 0); }
/* returns rough sin(x) approximation. the period is 2 (real sin has 2*Pi period). */ inline vec_float4 fast_sinf(vec_float4 x) { vec_uint4 xi = (vec_uint4)x; xi = spu_and(xi, 0x7fffffffU); vec_uint4 ui = spu_convtu((vec_float4)xi, 0); vec_float4 neg = (vec_float4)spu_sl(ui, 31u); vec_float4 t = spu_splats(.5f) - (((vec_float4)xi) - spu_convtf(ui, 0)); vec_float4 s = spu_splats(1.f) - spu_splats(4.f)*t*t; return spu_or(s, neg); }
inline void merge_cache_blocks(RenderableCacheLine* cache) { vec_uchar16 next = cache->chunkNext; for (;;) { vec_uchar16 nextnext = spu_shuffle(next, next, next); vec_uchar16 nextmask = spu_and(next, spu_splats((unsigned char)CHUNKNEXT_MASK)); vec_ushort8 firstblock0 = spu_cmpeq( cache->chunkStart[0], 0); vec_ushort8 firstblock1 = spu_cmpeq( cache->chunkStart[1], 0); // change next to word offset, note we don't care what the low bit shifted in is vec_uchar16 firstshuf = (vec_uchar16) spu_sl( (vec_ushort8)nextmask, 1 ); vec_uchar16 first = (vec_uchar16) spu_shuffle( firstblock0, firstblock1, firstshuf ); vec_ushort8 tri0 = cache->chunkTriangle[0]; vec_ushort8 tri1 = cache->chunkTriangle[1]; vec_uchar16 trishufhi = spu_or ( firstshuf, spu_splats((unsigned char) 1)); vec_uchar16 trishuflo = spu_and( firstshuf, spu_splats((unsigned char) 254)); vec_ushort8 ntri0 = spu_shuffle( tri0, tri1, spu_shuffle( trishuflo, trishufhi, SHUF0 ) ); vec_ushort8 ntri1 = spu_shuffle( tri0, tri1, spu_shuffle( trishuflo, trishufhi, SHUF1 ) ); vec_ushort8 trieq0 = spu_cmpeq( tri0, ntri0 ); vec_ushort8 trieq1 = spu_cmpeq( tri1, ntri1 ); vec_uchar16 trieq = (vec_uchar16) spu_shuffle( trieq0, trieq1, MERGE ); vec_uchar16 combi = spu_orc(first, trieq); vec_uchar16 canmerge = spu_cmpgt( spu_nor(spu_or(next, nextnext), combi), 256-CHUNKNEXT_BUSY_BIT ); vec_uint4 gather = spu_gather( canmerge ); vec_uint4 mergeid = spu_sub( spu_cntlz( gather ), spu_promote((unsigned int)16, 0)); if( !spu_extract(gather, 0) ) { return; } // unsigned int firstchunk = spu_extract(mergeid, 0); // unsigned int nextchunk = cache->chunkNextArray[firstchunk]; vec_uint4 v_chunkNext = (vec_uint4) si_rotqby( (qword) next, (qword) spu_add(mergeid,13) ); vec_uint4 v_chunkNextNext = (vec_uint4) si_rotqby( (qword) next, (qword) spu_add(v_chunkNext,13) ); // cache->chunkNextArray[firstchunk] = cache->chunkNextArray[nextchunk]; next = spu_shuffle( (vec_uchar16) v_chunkNextNext, next, (vec_uchar16) si_cbd( (qword) mergeid, 0 ) ); // cache->chunkNextArray[nextchunk] = CHUNKNEXT_FREE_BLOCK; next = spu_shuffle( spu_splats( (unsigned char) CHUNKNEXT_FREE_BLOCK), next, (vec_uchar16) si_cbd( (qword) v_chunkNext, 0 ) ); // this is for debug use only, it's not really needed... // cache->chunkStartArray[nextchunk] = -1; cache->chunkStartArray[ spu_extract(v_chunkNext,0) & 255 ] = -1; cache->chunkNext = next; } }
unsigned int __mfc_multi_tag_release (unsigned int first_tag, unsigned int number_of_tags) { vector unsigned int table_copy, tmp, tmp1; vector unsigned int one = (vector unsigned int) { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; vector unsigned int is_invalid; unsigned int last_tag; vector unsigned int has_been_reserved; last_tag = first_tag + number_of_tags; table_copy = spu_sl (one, number_of_tags); table_copy = spu_rl (table_copy, -last_tag); table_copy = spu_xor (table_copy, -1); /* Make sure the tags are in range and valid. */ tmp = spu_cmpgt (spu_promote(last_tag, 0), 32); tmp1 = spu_cmpgt (spu_promote(number_of_tags, 0), 32); is_invalid = spu_cmpgt (spu_promote(first_tag, 0), 31); /* All bits are set to 1 if invalid, 0 if valid. */ is_invalid = spu_or (tmp, is_invalid); is_invalid = spu_or (tmp1, is_invalid); /* check whether these tags have been reserved */ tmp = spu_rlmask (one, (int)-number_of_tags); tmp1 = spu_sl (__mfc_tag_table, first_tag); has_been_reserved = spu_cmpgt(tmp1, tmp); is_invalid = spu_or (has_been_reserved, is_invalid); table_copy = spu_sel (__mfc_tag_table, table_copy, table_copy); __mfc_tag_table = spu_sel (table_copy, __mfc_tag_table, is_invalid); return spu_extract (is_invalid, 0); }
void draw_frame(uint64_t buf_ea) { vec_uint4 buf[2*1920/4]; int row, col, i, tag = 0; float step = 4.0f/spu.width*spu.zoom; float xbeg = spu.xc - spu.width*step*0.5f; vec_float4 vxbeg = spu_splats(xbeg) + spu_splats(step) * (vec_float4) { 0.f,1.f,2.f,3.f }; vec_float4 xstep = spu_splats(step)*spu_splats(4.f); vec_float4 vyp = spu_splats(spu.yc - spu.height*step*0.5f + step*spu.rank); const vec_float4 vinc = spu_splats(spu.count * step); const vec_float4 esc2 = spu_splats(BAILOUT*BAILOUT); #if BAILBITS != 1 const vec_float4 esc21 = spu_splats(4.f/(BAILOUT*BAILOUT)); #endif const vec_float4 two = spu_splats(2.f); const vec_float4 zero = spu_splats(0.f); const vec_float4 colsc = spu_splats(255.f); const vec_float4 ccr = spu_splats(4.f*BAILOUT/(3.5f*3.141592654f)); const vec_float4 ccg = spu_splats(4.f*BAILOUT/(5.f*3.141592654f)); const vec_float4 ccb = spu_splats(4.f*BAILOUT/(9.f*3.141592654f)); vec_float4 x, y, x2, y2, m2, vxp; vec_uint4 cmp, inc; vec_uint4 vi; vec_uint4 *p, *b; vec_float4 co; /* Process the full image. As there are 6 SPUs working in parallel, each with * a different rank from 0 to 5, each SPU processes only the line numbers: * rank, rank+6, rank+12, ... * The program uses a SPU DMA programming technique known as "double buffering", * where the previously generated line is transmitted to main memory while we * compute the next one, hence the need for a local buffer containing two lines. */ for (row = spu.rank; row < spu.height; row += spu.count) { /* Pixel buffer address (in local memory) of the next line to be drawn */ b = p = buf + ((1920/4)&-tag); vxp = vxbeg; /* first four x coordinates */ /* Process a whole screen line by packets of 4 pixels */ for (col = spu.width/4; col > 0 ; col--) { vi = spu_splats(0u); x = vxp; y = vyp; i = 0; cmp = spu_splats(-1u); inc = spu_splats(1u); m2 = zero; /* This loop processes the Mandelbrot suite for the four complex numbers * whose real part are the components of the x vector, and the imaginary * part are in y (as we process the same line, all initial values of y * are equal). * We perform loop unrolling for SPU performance optimization reasons, * hence the 4x replication of the same computation block. */ do { x2 = x*x; y2 = y*y; m2 = spu_sel(m2, x2+y2, cmp); cmp = spu_cmpgt(esc2, m2); inc = spu_and(inc, cmp); /* increment the iteration count only if */ vi = vi + inc; /* we're still inside the bailout radius */ y = two*x*y + vyp; x = x2-y2 + vxp; x2 = x*x; y2 = y*y; m2 = spu_sel(m2, x2+y2, cmp); cmp = spu_cmpgt(esc2, m2); inc = spu_and(inc, cmp); vi = vi + inc; y = two*x*y + vyp; x = x2-y2 + vxp; x2 = x*x; y2 = y*y; m2 = spu_sel(m2, x2+y2, cmp); cmp = spu_cmpgt(esc2, m2); inc = spu_and(inc, cmp); vi = vi + inc; y = two*x*y + vyp; x = x2-y2 + vxp; x2 = x*x; y2 = y*y; m2 = spu_sel(m2, x2+y2, cmp); cmp = spu_cmpgt(esc2, m2); inc = spu_and(inc, cmp); vi = vi + inc; y = two*x*y + vyp; x = x2-y2 + vxp; i += 4; } /* Exit the loop only if the iteration limit of 128 has been reached, * or all current four points are outside the bailout radius. * The __builtin_expect(xxx, 1) construct hints the compiler that the xxx * test has greater chance of being true (1), so a branch hinting * instruction is inserted into the binary code to make the conditional * branch faster in most cases (except the last one when we exit the * loop). This results in performance increase. */ while (__builtin_expect((i < 128) & (si_to_int((qword)spu_gather(cmp)) != 0), 1)); /* smooth coloring: compute the fractional part */ co = spu_convtf(vi, 0) + spu_splats(1.f); co -= fast_logf(fast_logf(m2) * spu_splats(.5f)); #if BAILBITS != 1 co = spu_re(spu_rsqrte(co*esc21)); #endif /* Compute the red, green an blue pixel components */ vec_uint4 cr = spu_convtu(mcos(co * ccr) * colsc, 0); vec_uint4 cg = spu_convtu(mcos(co * ccg) * colsc, 0); vec_uint4 cb = spu_convtu(mcos(co * ccb) * colsc, 0); /* Put the 4 pixel values in the buffer */ *p++ = (spu_sl(cr, 16) | spu_sl(cg, 8) | cb) & ~-inc; vxp += xstep; } /* double-buffered dma: initiate a dma transfer of last computed scanline * then wait for completion of the second last transfer (previous computed * line). This is done by changing the tag value. */ mfc_put(b, buf_ea+(spu.width*4)*row, spu.width*4, tag, 0, 0); tag = 1 - tag; wait_for_completion(tag); vyp += vinc; } /* wait for completion of last sent image line */ wait_for_completion(1-tag); }
vector double __divv2df3 (vector double a_in, vector double b_in) { /* Variables */ vec_int4 exp, exp_bias; vec_uint4 no_underflow, overflow; vec_float4 mant_bf, inv_bf; vec_ullong2 exp_a, exp_b; vec_ullong2 a_nan, a_zero, a_inf, a_denorm, a_denorm0; vec_ullong2 b_nan, b_zero, b_inf, b_denorm, b_denorm0; vec_ullong2 nan; vec_uint4 a_exp, b_exp; vec_ullong2 a_mant_0, b_mant_0; vec_ullong2 a_exp_1s, b_exp_1s; vec_ullong2 sign_exp_mask; vec_double2 a, b; vec_double2 mant_a, mant_b, inv_b, q0, q1, q2, mult; /* Constants */ vec_uint4 exp_mask_u32 = spu_splats((unsigned int)0x7FF00000); vec_uchar16 splat_hi = (vec_uchar16) { 0,1,2,3, 0,1,2,3, 8, 9,10,11, 8,9,10,11 }; vec_uchar16 swap_32 = (vec_uchar16) { 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 }; vec_ullong2 exp_mask = spu_splats(0x7FF0000000000000ULL); vec_ullong2 sign_mask = spu_splats(0x8000000000000000ULL); vec_float4 onef = spu_splats(1.0f); vec_double2 one = spu_splats(1.0); vec_double2 exp_53 = (vec_double2)spu_splats(0x0350000000000000ULL); sign_exp_mask = spu_or(sign_mask, exp_mask); /* Extract the floating point components from each of the operands including * exponent and mantissa. */ a_exp = (vec_uint4)spu_and((vec_uint4)a_in, exp_mask_u32); a_exp = spu_shuffle(a_exp, a_exp, splat_hi); b_exp = (vec_uint4)spu_and((vec_uint4)b_in, exp_mask_u32); b_exp = spu_shuffle(b_exp, b_exp, splat_hi); a_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)a_in, sign_exp_mask), 0); a_mant_0 = spu_and(a_mant_0, spu_shuffle(a_mant_0, a_mant_0, swap_32)); b_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)b_in, sign_exp_mask), 0); b_mant_0 = spu_and(b_mant_0, spu_shuffle(b_mant_0, b_mant_0, swap_32)); a_exp_1s = (vec_ullong2)spu_cmpeq(a_exp, exp_mask_u32); b_exp_1s = (vec_ullong2)spu_cmpeq(b_exp, exp_mask_u32); /* Identify all possible special values that must be accommodated including: * +-denorm, +-0, +-infinity, and NaNs. */ a_denorm0= (vec_ullong2)spu_cmpeq(a_exp, 0); a_nan = spu_andc(a_exp_1s, a_mant_0); a_zero = spu_and (a_denorm0, a_mant_0); a_inf = spu_and (a_exp_1s, a_mant_0); a_denorm = spu_andc(a_denorm0, a_zero); b_denorm0= (vec_ullong2)spu_cmpeq(b_exp, 0); b_nan = spu_andc(b_exp_1s, b_mant_0); b_zero = spu_and (b_denorm0, b_mant_0); b_inf = spu_and (b_exp_1s, b_mant_0); b_denorm = spu_andc(b_denorm0, b_zero); /* Scale denorm inputs to into normalized numbers by conditionally scaling the * input parameters. */ a = spu_sub(spu_or(a_in, exp_53), spu_sel(exp_53, a_in, sign_mask)); a = spu_sel(a_in, a, a_denorm); b = spu_sub(spu_or(b_in, exp_53), spu_sel(exp_53, b_in, sign_mask)); b = spu_sel(b_in, b, b_denorm); /* Extract the divisor and dividend exponent and force parameters into the signed * range [1.0,2.0) or [-1.0,2.0). */ exp_a = spu_and((vec_ullong2)a, exp_mask); exp_b = spu_and((vec_ullong2)b, exp_mask); mant_a = spu_sel(a, one, (vec_ullong2)exp_mask); mant_b = spu_sel(b, one, (vec_ullong2)exp_mask); /* Approximate the single reciprocal of b by using * the single precision reciprocal estimate followed by one * single precision iteration of Newton-Raphson. */ mant_bf = spu_roundtf(mant_b); inv_bf = spu_re(mant_bf); inv_bf = spu_madd(spu_nmsub(mant_bf, inv_bf, onef), inv_bf, inv_bf); /* Perform 2 more Newton-Raphson iterations in double precision. The * result (q1) is in the range (0.5, 2.0). */ inv_b = spu_extend(inv_bf); inv_b = spu_madd(spu_nmsub(mant_b, inv_b, one), inv_b, inv_b); q0 = spu_mul(mant_a, inv_b); q1 = spu_madd(spu_nmsub(mant_b, q0, mant_a), inv_b, q0); /* Determine the exponent correction factor that must be applied * to q1 by taking into account the exponent of the normalized inputs * and the scale factors that were applied to normalize them. */ exp = spu_rlmaska(spu_sub((vec_int4)exp_a, (vec_int4)exp_b), -20); exp = spu_add(exp, (vec_int4)spu_add(spu_and((vec_int4)a_denorm, -0x34), spu_and((vec_int4)b_denorm, 0x34))); /* Bias the quotient exponent depending on the sign of the exponent correction * factor so that a single multiplier will ensure the entire double precision * domain (including denorms) can be achieved. * * exp bias q1 adjust exp * ===== ======== ========== * positive 2^+65 -65 * negative 2^-64 +64 */ exp_bias = spu_xor(spu_rlmaska(exp, -31), 64); exp = spu_sub(exp, exp_bias); q1 = spu_sel(q1, (vec_double2)spu_add((vec_int4)q1, spu_sl(exp_bias, 20)), exp_mask); /* Compute a multiplier (mult) to applied to the quotient (q1) to produce the * expected result. On overflow, clamp the multiplier to the maximum non-infinite * number in case the rounding mode is not round-to-nearest. */ exp = spu_add(exp, 0x3FF); no_underflow = spu_cmpgt(exp, 0); overflow = spu_cmpgt(exp, 0x7FE); exp = spu_and(spu_sl(exp, 20), (vec_int4)no_underflow); exp = spu_and(exp, (vec_int4)exp_mask); mult = spu_sel((vec_double2)exp, (vec_double2)(spu_add((vec_uint4)exp_mask, -1)), (vec_ullong2)overflow); /* Handle special value conditions. These include: * * 1) IF either operand is a NaN OR both operands are 0 or INFINITY THEN a NaN * results. * 2) ELSE IF the dividend is an INFINITY OR the divisor is 0 THEN a INFINITY results. * 3) ELSE IF the dividend is 0 OR the divisor is INFINITY THEN a 0 results. */ mult = spu_andc(mult, (vec_double2)spu_or(a_zero, b_inf)); mult = spu_sel(mult, (vec_double2)exp_mask, spu_or(a_inf, b_zero)); nan = spu_or(a_nan, b_nan); nan = spu_or(nan, spu_and(a_zero, b_zero)); nan = spu_or(nan, spu_and(a_inf, b_inf)); mult = spu_or(mult, (vec_double2)nan); /* Scale the final quotient */ q2 = spu_mul(q1, mult); return (q2); }
void process_render_tasks(unsigned long eah_render_tasks, unsigned long eal_render_tasks) { const vec_uchar16 SHUFFLE_MERGE_BYTES = (vec_uchar16) { // merge lo bytes from unsigned shorts (array) 1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31 }; const vec_uchar16 SHUFFLE_GET_BUSY_WITH_ONES = (vec_uchar16) { // get busy flag with ones in unused bytes 0xc0, 0xc0, 2, 3, 0xc0,0xc0,0xc0,0xc0, 0xc0,0xc0,0xc0,0xc0 }; const vec_uchar16 ZERO_BYTES = (vec_uchar16) spu_splats(0); char trianglebuffer[ 256 + TRIANGLE_MAX_SIZE ]; char sync_buffer[128+127]; void* aligned_sync_buffer = (void*) ( ((unsigned long)sync_buffer+127) & ~127 ); RenderableCacheLine* cache = (RenderableCacheLine*) aligned_sync_buffer; unsigned long long cache_ea; spu_mfcdma64(&cache_ea, eah_render_tasks, eal_render_tasks, sizeof(cache_ea), 0, MFC_GET_CMD); mfc_write_tag_mask(1<<0); mfc_read_tag_status_all(); while (cache_ea) { // terminate immediately if possible if (spu_stat_in_mbox()) return; // read the cache line spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_GETLLAR_CMD); spu_readch(MFC_RdAtomicStat); unsigned int endTriangle = cache->endTriangle; vec_ushort8 testTriangle = spu_splats((unsigned short) endTriangle); // first look for short chunks vec_uchar16 next = cache->chunkNext; vec_uchar16 nextmask = spu_and(next, spu_splats((unsigned char)CHUNKNEXT_MASK)); // change next to word offset, note we don't care what the low bit shifted in is vec_uchar16 firstshuf = (vec_uchar16) spu_sl( (vec_ushort8)nextmask, 1 ); vec_uchar16 trishufhi = spu_or ( firstshuf, spu_splats((unsigned char) 1)); vec_uchar16 trishuflo = spu_and( firstshuf, spu_splats((unsigned char) 254)); vec_ushort8 start0 = cache->chunkStart[0]; vec_ushort8 start1 = cache->chunkStart[1]; vec_ushort8 nstart0 = spu_shuffle( start0, start1, spu_shuffle( trishuflo, trishufhi, SHUF0 ) ); vec_ushort8 nstart1 = spu_shuffle( start0, start1, spu_shuffle( trishuflo, trishufhi, SHUF1 ) ); vec_ushort8 starteq0 = spu_cmpeq( nstart0, spu_splats((unsigned short)0) ); vec_ushort8 starteq1 = spu_cmpeq( nstart1, spu_splats((unsigned short)0) ); vec_ushort8 end0 = spu_sel( nstart0, spu_splats((unsigned short)4096), starteq0); vec_ushort8 end1 = spu_sel( nstart1, spu_splats((unsigned short)4096), starteq1); vec_ushort8 len0 = spu_sub( end0, start0); vec_ushort8 len1 = spu_sub( end1, start1); vec_ushort8 small0 = spu_cmpgt( spu_splats((unsigned short)17), len0); vec_ushort8 small1 = spu_cmpgt( spu_splats((unsigned short)17), len1); vec_uchar16 small = (vec_uchar16) spu_shuffle( small0, small1, MERGE ); vec_uint4 smallChunkGather = spu_gather(small); // check to see if chunk is already at the last triangle vec_uint4 doneChunkGather = spu_gather( (vec_uchar16) spu_shuffle( (vec_uchar16) spu_cmpeq(testTriangle, cache->chunkTriangle[0]), (vec_uchar16) spu_cmpeq(testTriangle, cache->chunkTriangle[1]), SHUFFLE_MERGE_BYTES) ); // check if the chunk is free vec_uint4 freeChunkGather = spu_gather( spu_cmpeq( spu_splats( (unsigned char) CHUNKNEXT_FREE_BLOCK ), cache->chunkNext ) ); // check to see if the chunk is being processed vec_uint4 busyChunkGather = spu_gather( spu_cmpgt( cache->chunkNext, //spu_and(cache->chunkNext, CHUNKNEXT_MASK), spu_splats( (unsigned char) (CHUNKNEXT_BUSY_BIT-1) ) ) ); // doneChunkGather, freeChunkGather, busyChunkGather - rightmost 16 bits of word 0 // note that if freeChunkGather is true then busyChunkGather must also be true // done=false, free=false, busy=false -> can process // free=false, busy=false -> can be merged // decide which chunk to process vec_uint4 mayProcessGather = spu_nor( doneChunkGather, busyChunkGather ); vec_uint4 mayProcessShortGather = spu_and( mayProcessGather, smallChunkGather ); vec_uint4 shortSelMask = spu_cmpeq( mayProcessShortGather, spu_splats(0U) ); vec_uint4 mayProcessSelection = spu_sel( mayProcessShortGather, mayProcessGather, shortSelMask ); /* if (!spu_extract(shortSelMask, 0)) printf("taken short: may=%04x short=%04x mayshort=%04x mask=%04x sel=%04x\n", spu_extract(mayProcessGather, 0) & 0xffff, spu_extract(smallChunkGather, 0), spu_extract(mayProcessShortGather, 0), spu_extract(shortSelMask, 0) & 0xffff, spu_extract(mayProcessSelection, 0) & 0xffff ); */ vec_uint4 mayProcessBits = spu_sl( mayProcessSelection, 16); unsigned int chunkToProcess = spu_extract( spu_cntlz( mayProcessBits ), 0); unsigned int freeChunk = spu_extract( spu_cntlz( spu_sl( freeChunkGather, 16 ) ), 0); // if there's nothing to process, try the next cache line in the rendering tasks list if (!spu_extract(mayProcessBits, 0)) { trynextcacheline: cache_ea = cache->next; // sleep(); continue; } unsigned int chunkStart = cache->chunkStartArray [chunkToProcess]; unsigned int chunkTriangle = cache->chunkTriangleArray[chunkToProcess]; unsigned int chunkNext = cache->chunkNextArray [chunkToProcess] & CHUNKNEXT_MASK; unsigned int chunkEnd = (cache->chunkStartArray [chunkNext]-1) & (NUMBER_OF_TILES-1); unsigned int chunkLength = 1 + chunkEnd-chunkStart; // only need an extra block if the block is especially long if (chunkLength <= NUMBER_OF_TILES_PER_CHUNK) { freeChunk = 32; } // mark this block as busy cache->chunkNextArray[chunkToProcess] |= CHUNKNEXT_BUSY_BIT; // if there's at least one free chunk, claim it if (freeChunk != 32) { cache->chunkNextArray[freeChunk] = CHUNKNEXT_RESERVED; cache->chunkTriangleArray[freeChunk] = chunkTriangle; } // write the cache line back spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_PUTLLC_CMD); if (spu_readch(MFC_RdAtomicStat) & MFC_PUTLLC_STATUS) continue; #ifdef INFO printf("[%d] Claimed chunk %d (%d-%d len %d) at tri %x end %x with free chunk %d\n", _SPUID, chunkToProcess, chunkStart, chunkEnd, chunkLength, chunkTriangle, endTriangle, freeChunk!=32 ? freeChunk : -1 ); // debug_render_tasks(cache); #endif Triangle* triangle; int firstTile; do { // read the triangle data for the current triangle unsigned int extra = chunkTriangle & 127; unsigned long long trianglebuffer_ea = cache_ea + TRIANGLE_OFFSET_FROM_CACHE_LINE + (chunkTriangle & ~127); triangle = (Triangle*) (trianglebuffer+extra); unsigned int length = (extra + TRIANGLE_MAX_SIZE + 127) & ~127; // ensure DMA slot available do {} while (!spu_readchcnt(MFC_Cmd)); spu_mfcdma64(trianglebuffer, mfc_ea2h(trianglebuffer_ea), mfc_ea2l(trianglebuffer_ea), length, 0, MFC_GET_CMD); mfc_write_tag_mask(1<<0); mfc_read_tag_status_all(); // get the triangle deltas firstTile = findFirstTriangleTile(triangle, chunkStart, chunkEnd); if (firstTile>=0) break; // no match, try next triangle chunkTriangle = triangle->next_triangle; } while (chunkTriangle != endTriangle); // if we actually have something to process... if (firstTile>=0) { // the "normal" splitting will now become: // chunkStart .. (firstTile-1) -> triangle->next_triangle // firstTile .. (firstTile+NUM-1) -> chunkTriangle (BUSY) // (firstTile+NUM) .. chunkEnd -> chunkTriangle (FREE) int tailChunk; int thisChunk; int nextBlockStart; int thisBlockStart; int realBlockStart; do { retry: // read the cache line spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_GETLLAR_CMD); spu_readch(MFC_RdAtomicStat); // calculate start of next block nextBlockStart = firstTile + NUMBER_OF_TILES_PER_CHUNK; if (nextBlockStart > chunkEnd) nextBlockStart = chunkEnd+1; // calculate start of block to mark as busy thisBlockStart = nextBlockStart - NUMBER_OF_TILES_PER_CHUNK; if (thisBlockStart < chunkStart) thisBlockStart = chunkStart; realBlockStart = thisBlockStart; #ifdef INFO printf("[%d] nextBlockStart=%d, realBlockStart=%d, thisBlockStart=%d, chunkStart=%d\n", _SPUID, nextBlockStart, realBlockStart, thisBlockStart, chunkStart); #endif // allocate some more free chunks vec_uint4 freeChunkGather2 = spu_sl(spu_gather(spu_cmpeq( spu_splats((unsigned char)CHUNKNEXT_FREE_BLOCK), cache->chunkNext)), 16); unsigned int freeChunk2 = spu_extract(spu_cntlz(freeChunkGather2), 0); if (freeChunk == 32) { // if we didn't have one before, try again freeChunk = freeChunk2; // and try to get the second one freeChunkGather2 = spu_andc( freeChunkGather2, spu_promote(0x80000000>>freeChunk2, 0) ); freeChunk2 = spu_extract(spu_cntlz(freeChunkGather2), 0); } else { // speculatively clear the free chunk just in case we don't need it cache->chunkNextArray[freeChunk] = CHUNKNEXT_FREE_BLOCK; } #ifdef INFO printf("[%d] Free chunks %d and %d, cN=%d, nBS=%d, cE=%d, tBS=%d, cS=%d\n", _SPUID, freeChunk, freeChunk2, chunkNext, nextBlockStart, chunkEnd, thisBlockStart, chunkStart ); #endif // mark region after as available for processing if required if (nextBlockStart < chunkEnd) { if (freeChunk==32) { // if no free chunk, relinquish entire block and write back cache->chunkNextArray[chunkToProcess] = chunkNext; spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_PUTLLC_CMD); // if writeback failed, we *might* have a free block, retry if (spu_readch(MFC_RdAtomicStat) & MFC_PUTLLC_STATUS) goto retry; // otherwise give up and try the next cache line goto trynextcacheline; } cache->chunkStartArray[freeChunk] = nextBlockStart; cache->chunkNextArray[freeChunk] = chunkNext; cache->chunkTriangleArray[freeChunk] = chunkTriangle; cache->chunkNextArray[chunkToProcess] = freeChunk | CHUNKNEXT_BUSY_BIT; tailChunk = freeChunk; #ifdef INFO printf("[%d] Insert tail, tailChunk=%d, chunkNext=%d, chunkToProcess=%d\n", _SPUID, tailChunk, chunkNext, chunkToProcess); debug_render_tasks(cache); #endif } else { // we're gonna use freeChunk2 for the "in front" block, as we've not // used freeChunk, let's use it as it's more likely to have a free chunk freeChunk2 = freeChunk; tailChunk = chunkNext; } // mark region before as available if required and possible thisChunk = chunkToProcess; if (thisBlockStart > chunkStart) { if (freeChunk2 != 32) { // mark this region as busy cache->chunkStartArray[freeChunk2]=thisBlockStart; cache->chunkNextArray[freeChunk2]=tailChunk | CHUNKNEXT_BUSY_BIT; cache->chunkTriangleArray[freeChunk2]=chunkTriangle; // mark region before as available for processing cache->chunkNextArray[chunkToProcess]=freeChunk2; cache->chunkTriangleArray[chunkToProcess]=triangle->next_triangle; thisChunk = freeChunk2; #ifdef INFO printf("[%d] Insert new head, tailChunk=%d, chunkNext=%d, thisChunk=%d\n", _SPUID, tailChunk, chunkNext, thisChunk); debug_render_tasks(cache); #endif } else { // need to keep whole block, update info and mark bust cache->chunkTriangleArray[chunkToProcess]=chunkTriangle; cache->chunkNextArray[chunkToProcess]=tailChunk | CHUNKNEXT_BUSY_BIT; realBlockStart = chunkStart; printf("[%d] Keep whole block, tailChunk=%d, chunkNext=%d, thisChunk=%d\n", _SPUID, tailChunk, chunkNext, thisChunk); debug_render_tasks(cache); #ifdef INFO #endif sleep(); } } // merge chunks merge_cache_blocks(cache); // write the cache line back spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_PUTLLC_CMD); } while (spu_readch(MFC_RdAtomicStat) & MFC_PUTLLC_STATUS); // finally after the write succeeded, update the variables chunkNext = tailChunk; chunkToProcess = thisChunk; chunkStart = firstTile; //thisBlockStart; chunkLength = nextBlockStart - firstTile; chunkEnd = chunkStart + chunkLength - 1; freeChunk = 32; // now we can process the block up to endTriangle initTileBuffers(thisBlockStart, chunkEnd); int ok=0; while (chunkTriangle != endTriangle) { #ifdef INFO printf("[%d] Processing chunk %d at %4d len %4d, triangle %04x first=%d tbs=%d\n", _SPUID, chunkToProcess, chunkStart, chunkLength, chunkTriangle, firstTile, thisBlockStart); #endif // and actually process that triangle on these chunks processTriangleChunks(triangle, cache, thisBlockStart, chunkEnd, chunkTriangle, ok); ok=1; #ifdef PAUSE sleep(); #endif // and advance to the next-triangle chunkTriangle = triangle->next_triangle; // this should only ever happen if we're running really low on cache line slots // basically, if we pick up a block with more than NUMBER_OF_TILES_PER_CHUNK and // there's no slot to store the pre-NUMBER_OF_TILES_PER_CHUNK tiles. // in this case, we process from thisBlockStart only (because we know that from // chunkStart to there has no result) and then we only process one triangle if (chunkStart != realBlockStart) { /* printf("[%d] chunkStart=%d != realBlockStart %d, chunkEnd=%d, " "firstTile=%d chunk=%d\n", _SPUID, chunkStart, realBlockStart, chunkEnd, firstTile, chunkToProcess); debug_render_tasks(cache); */ // abort the while loop break; } // read the next triangle unsigned int extra = chunkTriangle & 127; unsigned long long trianglebuffer_ea = cache_ea + TRIANGLE_OFFSET_FROM_CACHE_LINE + (chunkTriangle & ~127); triangle = (Triangle*) (trianglebuffer+extra); unsigned int length = (extra + TRIANGLE_MAX_SIZE + 127) & ~127; // ensure DMA slot available do {} while (!spu_readchcnt(MFC_Cmd)); spu_mfcdma64(trianglebuffer, mfc_ea2h(trianglebuffer_ea), mfc_ea2l(trianglebuffer_ea), length, 0, MFC_GET_CMD); mfc_write_tag_mask(1<<0); mfc_read_tag_status_all(); } // until chunkTriangle == endTriangle // flush any output buffers flushTileBuffers(thisBlockStart, chunkEnd); } // firstTile>=0
/* Scans the string pointed to by s for the character c and * returns a pointer to the last occurance of c. If * c is not found, then NULL is returned. */ char * strrchr(const char *s, int c) { int nskip; vec_uchar16 *ptr, data, vc; vec_uint4 cmp_c, cmp_0, cmp; vec_uint4 res_ptr, res_cmp; vec_uint4 mask, result; vec_uint4 one = spu_splats(0xffffU); /* Scan memory array a quadword at a time. Skip leading * mis-aligned bytes. */ ptr = (vec_uchar16 *)s; nskip = -((unsigned int)(ptr) & 15); mask = spu_rlmask(one, nskip); vc = spu_splats((unsigned char)(c)); data = *ptr++; ptr = (vec_uchar16 *)((unsigned int)ptr & ~15); cmp_c = spu_and(spu_gather(spu_cmpeq(data, vc)), mask); cmp_0 = spu_and(spu_gather(spu_cmpeq(data, 0)), mask); res_ptr = spu_splats(0U); res_cmp = spu_splats(0U); while (spu_extract(cmp_0, 0) == 0) { cmp = spu_cmpeq(cmp_c, 0); res_ptr = spu_sel(spu_promote((unsigned int)(ptr), 0), res_ptr, cmp); res_cmp = spu_sel(cmp_c, res_cmp, cmp); data = *ptr++; cmp_c = spu_gather(spu_cmpeq(data, vc)); cmp_0 = spu_gather(spu_cmpeq(data, 0)); cmp = spu_cmpeq(cmp_c, 0); } /* Compute the location of the last character before termination * character. * * First mask off compare results following the first termination character. */ mask = spu_sl(one, 31 - spu_extract(spu_cntlz(cmp_0), 0)); cmp_c = spu_and(cmp_c, mask); /* Conditionally update res_ptr and res_cmd if a match was found in the last * quadword. */ cmp = spu_cmpeq(cmp_c, 0); res_ptr = spu_sel(spu_promote((unsigned int)(ptr), 0), res_ptr, cmp); res_cmp = spu_sel(cmp_c, res_cmp, cmp); /* Bit reserve res_cmp for locating last occurance. */ mask = spu_cmpeq(res_cmp, 0); res_cmp = (vec_uint4)spu_maskb(spu_extract(res_cmp, 0)); res_cmp = spu_gather((vec_uchar16)spu_shuffle(res_cmp, res_cmp, VEC_LITERAL(vec_uchar16, 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0))); /* Compute the location (ptr) of the last occurance of c. If no * occurance was found (ie, element 0 of res_cmp == 0, then return * NULL. */ result = spu_sub(spu_add(res_ptr, 15), spu_cntlz(res_cmp)); result = spu_andc(result, mask); return ((char *)spu_extract(result, 0)); }