/**
* @v
* @vth
*/
inline vec_float4 updateF(vec_float4 v,vec_float4 vth)
{
vec_float4 updateF_rtn;
updateF_rtn = spu_sel(updateF_rtn,1.0,spu_cmpgt(spu_sub(v,vth),0.0));
updateF_rtn = spu_sel(updateF_rtn,0.0,spu_nand(spu_cmpgt(spu_sub(v,vth),0.0),spu_cmpgt(spu_sub(v,vth),0.0)));
return updateF_rtn;
}
inline void MinMaxBinCount3SIMD(aabb_t *aabb, minmaxbin_t *mmb, aabb_t *baabb)
{
vector float *baabb_min = (vector float*)baabb->min;
vector float *baabb_max = (vector float*)baabb->max;
vector float *aabb_min = (vector float*)aabb->min;
vector float *aabb_max = (vector float*)aabb->max;
vector float nbins = spu_splats((float)nsamplepoints);
vector float invnbins = spu_re(nbins);
nbins = spu_sub(nbins, spu_splats(1.0f));
vector float width = spu_abs(spu_sub(*baabb_max, *baabb_min));
vector float invdelta = spu_re(spu_mul(width, invnbins));
vector int minindex = GetBinSIMD(*baabb_min, *baabb_max, *aabb_min, invdelta, nbins);
vector int maxindex = GetBinSIMD(*baabb_min, *baabb_max, *aabb_max, invdelta, nbins);
mmb->minbins[minindex[0]].b[0]++;
mmb->minbins[minindex[1]].b[1]++;
mmb->minbins[minindex[2]].b[2]++;
mmb->maxbins[maxindex[0]].b[0]++;
mmb->maxbins[maxindex[1]].b[1]++;
mmb->maxbins[maxindex[2]].b[2]++;
}
vec_uint4 ulpDiff_f4(vec_float4 ref, vec_float4 vals) {
vec_int4 refi = (vec_int4)ref;
vec_int4 valsi = (vec_int4)vals;
vec_int4 diff = spu_sub(refi, valsi);
vec_int4 negdiff = spu_sub(spu_splats((int)0), diff);
return (vec_uint4)(spu_sel(negdiff, diff, spu_cmpgt(diff, 0)));
}
inline void AreaLeftRight(vector float aabb_min, vector float aabb_max, vector float side, vector float plane, vector float *aleft, vector float *aright)
{
vector float lside;
vector float rside;
lside = spu_abs(spu_sub(plane, aabb_min));
rside = spu_abs(spu_sub(aabb_max, plane));
*aleft = spu_mul(lside, side);
*aright = spu_mul(rside, side);
}
int num_in_buffer(int side){
volatile vector signed int *head_idx, *tail_idx;
int buffer_size;
if(side == OUT && mcb[am].local[OUT] < 255){
int parent_idx = mcb[am].local[OUT];
int side = (mcb[am].id+1)&1;
head_idx = &md[parent_idx].idx[side][HEAD];
tail_idx = &md[parent_idx].idx[side][TAIL];
buffer_size = mcb[parent_idx].buffer_size[side];
} else {
head_idx = &md[am].idx[side][HEAD];
tail_idx = &md[am].idx[side][TAIL];
buffer_size = mcb[am].buffer_size[side];
}
vector signed int diff = spu_sub(*head_idx,*tail_idx);
int num = spu_extract(diff,0);
if(num < 0)
num = num + buffer_size;
return num;
}
void check_pull_dma(int side){
// Check left
if(md[am].held_tag[side] < 32){
mfc_write_tag_mask( 1 << md[am].held_tag[side] );
int status = mfc_read_tag_status_immediate();
if(status){
// Update idx
md[am].idx[side][HEAD] = spu_add(md[am].idx[side][HEAD], md[am].num_waiting[side]);
vector signed int buffer_size = spu_splats(mcb[am].buffer_size[side] -1);
vector unsigned int cmp_v = spu_cmpgt(md[am].idx[side][HEAD], buffer_size);
vector signed int zeros = {0,0,0,0};
buffer_size = spu_add(buffer_size,1);
zeros = spu_sel(zeros,buffer_size,cmp_v);
md[am].idx[side][HEAD] = spu_sub(md[am].idx[side][HEAD],zeros);
md[am].num_pulled[side] += md[am].num_waiting[side];
md[am].num_waiting[side] = 0;
if(md[am].num_pulled[side] == mcb[am].data_size[side]){
md[am].mm_depleted[side] = 1;
}
// Release tag
mfc_tag_release( md[am].held_tag[side] );
md[am].held_tag[side] = 32;
}
}
}
inline void merge_cache_blocks(RenderableCacheLine* cache)
{
vec_uchar16 next = cache->chunkNext;
for (;;) {
vec_uchar16 nextnext = spu_shuffle(next, next, next);
vec_uchar16 nextmask = spu_and(next, spu_splats((unsigned char)CHUNKNEXT_MASK));
vec_ushort8 firstblock0 = spu_cmpeq( cache->chunkStart[0], 0);
vec_ushort8 firstblock1 = spu_cmpeq( cache->chunkStart[1], 0);
// change next to word offset, note we don't care what the low bit shifted in is
vec_uchar16 firstshuf = (vec_uchar16) spu_sl( (vec_ushort8)nextmask, 1 );
vec_uchar16 first = (vec_uchar16) spu_shuffle( firstblock0, firstblock1, firstshuf );
vec_ushort8 tri0 = cache->chunkTriangle[0];
vec_ushort8 tri1 = cache->chunkTriangle[1];
vec_uchar16 trishufhi = spu_or ( firstshuf, spu_splats((unsigned char) 1));
vec_uchar16 trishuflo = spu_and( firstshuf, spu_splats((unsigned char) 254));
vec_ushort8 ntri0 = spu_shuffle( tri0, tri1, spu_shuffle( trishuflo, trishufhi, SHUF0 ) );
vec_ushort8 ntri1 = spu_shuffle( tri0, tri1, spu_shuffle( trishuflo, trishufhi, SHUF1 ) );
vec_ushort8 trieq0 = spu_cmpeq( tri0, ntri0 );
vec_ushort8 trieq1 = spu_cmpeq( tri1, ntri1 );
vec_uchar16 trieq = (vec_uchar16) spu_shuffle( trieq0, trieq1, MERGE );
vec_uchar16 combi = spu_orc(first, trieq);
vec_uchar16 canmerge = spu_cmpgt( spu_nor(spu_or(next, nextnext), combi), 256-CHUNKNEXT_BUSY_BIT );
vec_uint4 gather = spu_gather( canmerge );
vec_uint4 mergeid = spu_sub( spu_cntlz( gather ), spu_promote((unsigned int)16, 0));
if( !spu_extract(gather, 0) ) {
return;
}
// unsigned int firstchunk = spu_extract(mergeid, 0);
// unsigned int nextchunk = cache->chunkNextArray[firstchunk];
vec_uint4 v_chunkNext = (vec_uint4) si_rotqby( (qword) next, (qword) spu_add(mergeid,13) );
vec_uint4 v_chunkNextNext = (vec_uint4) si_rotqby( (qword) next, (qword) spu_add(v_chunkNext,13) );
// cache->chunkNextArray[firstchunk] = cache->chunkNextArray[nextchunk];
next = spu_shuffle( (vec_uchar16) v_chunkNextNext, next, (vec_uchar16) si_cbd( (qword) mergeid, 0 ) );
// cache->chunkNextArray[nextchunk] = CHUNKNEXT_FREE_BLOCK;
next = spu_shuffle( spu_splats( (unsigned char) CHUNKNEXT_FREE_BLOCK), next, (vec_uchar16) si_cbd( (qword) v_chunkNext, 0 ) );
// this is for debug use only, it's not really needed...
// cache->chunkStartArray[nextchunk] = -1;
cache->chunkStartArray[ spu_extract(v_chunkNext,0) & 255 ] = -1;
cache->chunkNext = next;
}
}
void *
sbrk (ptrdiff_t increment)
{
static caddr_t heap_ptr = NULL;
caddr_t base;
vector unsigned int sp_reg, sp_delta;
vector unsigned int *sp_ptr;
caddr_t sps;
/* The stack pointer register. */
volatile register vector unsigned int sp_r1 __asm__("1");
if (heap_ptr == NULL)
heap_ptr = (caddr_t) & _end;
sps = (caddr_t) spu_extract (sp_r1, 0);
if (((int) sps - STACKSIZE - (int) heap_ptr) >= increment)
{
base = heap_ptr;
heap_ptr += increment;
sp_delta = (vector unsigned int) spu_insert (increment, spu_splats (0), 1);
/* Subtract sp_delta from the SP limit (word 1). */
sp_r1 = spu_sub (sp_r1, sp_delta);
/* Fix-up backchain. */
sp_ptr = (vector unsigned int *) spu_extract (sp_r1, 0);
do
{
sp_reg = *sp_ptr;
*sp_ptr = (vector unsigned int) spu_sub (sp_reg, sp_delta);
}
while ((sp_ptr = (vector unsigned int *) spu_extract (sp_reg, 0)));
return (base);
}
else
{
errno = ENOMEM;
return ((void *) -1);
}
}
inline vector int GetBinSIMD(vector float left, vector float right, vector float pos, vector float invdelta, vector float nbins)
{
pos = spu_min(pos, right);
pos = spu_max(pos, left);
vector float bin = spu_mul(spu_abs(spu_sub(pos, left)), invdelta);
bin = spu_min(bin, nbins);
bin = spu_max(spu_splats(0.0f), bin);
return spu_convts(bin, 0);
}
unsigned int
__mfc_multi_tag_reserve (unsigned int number_of_tags)
{
vector unsigned int table_copy;
vector unsigned int one = (vector unsigned int)
{ 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF };
vector unsigned int count_busy, is_valid;
vector unsigned int count_total;
vector unsigned int count_avail = (vector unsigned int) { 0, 0, 0, 0 };
vector unsigned int index = (vector unsigned int) { 0, 0, 0, 0 };
table_copy = __mfc_tag_table;
/* count_busy: number of consecutive busy tags
count_avail: number of consecutive free tags
table_copy: temporary copy of the tag table
count_total: sum of count_busy and count_avail
index: index of the current working tag */
do
{
table_copy = spu_sl (table_copy, count_avail);
count_busy = spu_cntlz (table_copy);
table_copy = spu_sl (table_copy, count_busy);
count_avail = spu_cntlz (spu_xor(table_copy, -1));
count_total = spu_add (count_busy, count_avail);
index = spu_add (index, count_total);
}
while (spu_extract (count_avail, 0) < number_of_tags
&& spu_extract (table_copy, 0) != 0);
index = spu_sub (index, count_avail);
/* is_valid is set to 0xFFFFFFFF if table_copy == 0, 0 otherwise. */
is_valid = spu_cmpeq (table_copy, 0);
index = spu_sel (index, is_valid, is_valid);
/* Now I need to actually mark the tags as used. */
table_copy = spu_sl (one, number_of_tags);
table_copy = spu_rl (table_copy, -number_of_tags - spu_extract (index, 0));
table_copy = spu_sel (table_copy, __mfc_tag_table, table_copy);
__mfc_tag_table = spu_sel (table_copy, __mfc_tag_table, is_valid);
return spu_extract (index, 0);
}
void _compute( unsigned int bs, unsigned int k, unsigned int row, vector float *BKJ, float *B, float *A )
{
vector float *BIJ, aik;
unsigned int i;
for( ; k < bs ; k++ )
{
aik = spu_splats( A[k*bs+row] );
BIJ = (vector float*)( B + ( k * bs ) );
// -----> For each vector in row
for( i = 0 ; i < bs / 4 ; i++ )
{
// BIJ[i] = BIJ[i] - BKJ[i] * aik;
BIJ[i] = spu_sub( BIJ[i], spu_mul( BKJ[i], aik ) );
}
}
}
unsigned int
__mfc_tag_reserve (void)
{
vector unsigned int mask = (vector unsigned int)
{ 0x80000000, 0x80000000, 0x80000000, 0x80000000 };
vector unsigned int count_zeros, is_valid;
vector signed int count_neg;
count_zeros = spu_cntlz (__mfc_tag_table);
count_neg = spu_sub (0, (vector signed int) count_zeros);
mask = spu_rlmask (mask, (vector signed int) count_neg);
__mfc_tag_table = spu_andc (__mfc_tag_table, mask);
is_valid = spu_cmpeq (count_zeros, 32);
count_zeros = spu_sel (count_zeros, is_valid, is_valid);
return spu_extract (count_zeros, 0);
}
void _compute2( unsigned int bs, unsigned int k, unsigned int row, vector float *BKJ, float *B, float *A, Functions_t *funcs )
{
vector float *BIJ, aik;
unsigned int i;
for( ; k < bs ; k++ )
{
aik = spu_splats( A[k*bs+row] );
BIJ = (vector float*)( B + ( k * bs ) );
funcs->printuint( 900000 + k );
// -----> For each vector in row
for( i = 0 ; i < bs / 4 ; i++ )
{
funcs->printfloatv( &aik );
funcs->printfloatv( &BIJ[i] );
funcs->printfloatv( &BKJ[i] );
// BIJ[i] = BIJ[i] - BKJ[i] * aik;
BIJ[i] = spu_sub( BIJ[i], spu_mul( BKJ[i], aik ) );
funcs->printfloatv( &BIJ[i] );
}
}
}
void merge_buffers(){
vector unsigned int cmp_v, cmp_v2;
const vector signed int one_at_0 = {1,0,0,0};
const vector signed int one_at_1 = {0,1,0,0};
const vector signed int one_at_2 = {0,0,1,0};
const vector signed int ones = {1,1,1,1};
const vector signed int zeros = {0,0,0,0};
const vector unsigned char cmp_v_shuffle_mask = {31,31,31,31,
31,31,31,31,
31,31,31,31,
31,31,31,31};
vector unsigned char rev_mask;
const vector unsigned char rev_left = {12,13,14,15,
8,9,10,11,
4,5,6,7,
0,1,2,3};
const vector unsigned char rev_right = {28,29,30,31,
24,25,26,27,
20,21,22,23,
16,17,18,19};
vector signed int *out_head_idx;
if(mcb[am].local[OUT] < 255){
int parent_idx = mcb[am].local[OUT];
int side = (mcb[am].id+1)&1;
out_head_idx = (vector signed int*) &md[parent_idx].idx[side][HEAD];
} else {
out_head_idx = (vector signed int*) &md[am].idx[OUT][HEAD];
}
vector signed int *left_tail_idx = (vector signed int*) &md[am].idx[LEFT][TAIL];
vector signed int *right_tail_idx = (vector signed int*) &md[am].idx[RIGHT][TAIL];
vector signed int size_v = {mcb[am].buffer_size[LEFT], mcb[am].buffer_size[RIGHT], mcb[am].buffer_size[OUT], 0};
vector signed int avail_v = {num_in_buffer(LEFT), num_in_buffer(RIGHT), num_free_in_buffer(OUT), 1};
vector signed int avail_before = { spu_extract(avail_v, 0), spu_extract(avail_v, 1), 0, 0 };
vector unsigned int avail = spu_gather( spu_cmpgt(avail_v, zeros) ); // avail = 0x0F if all avail_v > zeros
vector signed int *left, *right, *out;
left = (vector signed int*) &md[am].buffer[LEFT][ spu_extract(*left_tail_idx,0) ];
right = (vector signed int*) &md[am].buffer[RIGHT][ spu_extract(*right_tail_idx,0) ];
out = (vector signed int*) &md[am].buffer[OUT][ spu_extract(*out_head_idx,0) ];
#ifdef TRACE_TIME
dec_val2 = spu_read_decrementer();
#endif
while(spu_extract(avail,0) == 0x0F){
// cmp left and right to determine who gets eaten
cmp_v = spu_cmpgt(*left,*right);
cmp_v = spu_shuffle(cmp_v, cmp_v, cmp_v_shuffle_mask);
// cmp_v = {FFFF,FFFF,FFFF,FFFF} if left[3] > right[3]
*out = spu_sel(*left,*right,cmp_v);
rev_mask = spu_sel(rev_right,rev_left,(vector unsigned char)cmp_v);
*left = spu_shuffle(*left,*right,rev_mask);
// data to be sorted is now in out and left, left in descending order
sort_vectors(out,left);
// update index of the used side
if( spu_extract(cmp_v,0) ){
// left[3] > right[3]
*right_tail_idx = spu_add(*right_tail_idx,ones);
avail_v = spu_sub(avail_v, one_at_1);
right++;
// modulus hack
cmp_v2 = spu_cmpeq(*right_tail_idx, size_v);
if( __builtin_expect( spu_extract(cmp_v2,0) ,0) ){
*right_tail_idx = zeros;
right = (vector signed int*) &md[am].buffer[RIGHT][0];
}
} else {
*right = *left;
*left_tail_idx = spu_add(*left_tail_idx,ones);
avail_v = spu_sub(avail_v, one_at_0);
left++;
// modulus hack
cmp_v2 = spu_cmpeq(*left_tail_idx, size_v);
if( __builtin_expect( spu_extract(cmp_v2,0) ,0) ){
*left_tail_idx = zeros;
left = (vector signed int*) &md[am].buffer[LEFT][0];
}
}
// update out head idx
*out_head_idx = spu_add(*out_head_idx,ones);
avail_v = spu_sub(avail_v, one_at_2);
out++;
// modulus hack
cmp_v2 = spu_cmpeq(*out_head_idx, size_v);
if( __builtin_expect(spu_extract(cmp_v2,0),0) ){
out = (vector signed int*) &md[am].buffer[OUT][0];
*out_head_idx = zeros;
}
// is there data still available?
avail = spu_gather(spu_cmpgt(avail_v, zeros));
}
#ifdef TRACE_TIME
merge_loop_ticks += -(spu_read_decrementer() - dec_val2);
#endif
// how much got produced?
vector signed int consumed = spu_sub(avail_before, avail_v);
int consumed_left = spu_extract(consumed, 0);
int consumed_right = spu_extract(consumed, 1);
if(consumed_left)
update_tail(LEFT);
if(consumed_right)
update_tail(RIGHT);
md[am].consumed[LEFT] += consumed_left;
md[am].consumed[RIGHT] += consumed_right;
if(md[am].consumed[LEFT] == mcb[am].data_size[LEFT])
md[am].depleted[LEFT] = 1;
if(md[am].consumed[RIGHT] == mcb[am].data_size[RIGHT])
md[am].depleted[RIGHT] = 1;
if(mcb[am].local[OUT] < 255 && md[am].depleted[LEFT] && md[am].depleted[RIGHT]){
md[am].done = 1;
--num_active_mergers;
}
}
void* libvector_16s_vector_subtract_unaligned(void* target, void* src0, void* src1, unsigned int num_bytes){
//loop iterator i
int i = 0;
void* retval = target;
//put the target and source addresses into qwords
vector unsigned int address_counter_tgt = {(unsigned int)target, 0, 0, 0};
vector unsigned int address_counter_src0 = {(unsigned int)src0, 0, 0 ,0};
vector unsigned int address_counter_src1 = {(unsigned int)src1, 0, 0, 0};
//create shuffle masks
//shuffle mask building blocks:
//all from the first vector
vector unsigned char oneup = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
//all from the second vector
vector unsigned char second_oneup = {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f};
//gamma: second half of the second, first half of the first, break at (unsigned int)src0%16
vector unsigned char src_cmp = spu_splats((unsigned char)((unsigned int)src0%16));
vector unsigned char gt_res = spu_cmpgt(oneup, src_cmp);
vector unsigned char eq_res = spu_cmpeq(oneup, src_cmp);
vector unsigned char cmp_res = spu_or(gt_res, eq_res);
vector unsigned char sixteen_uchar = spu_splats((unsigned char)16);
vector unsigned char phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_gamma = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
shuffle_mask_gamma = spu_rlqwbyte(shuffle_mask_gamma, (unsigned int)src0%16);
//eta: second half of the second, first half of the first, break at (unsigned int)src1%16
src_cmp = spu_splats((unsigned char)((unsigned int)src1%16));
gt_res = spu_cmpgt(oneup, src_cmp);
eq_res = spu_cmpeq(oneup, src_cmp);
cmp_res = spu_or(gt_res, eq_res);
sixteen_uchar = spu_splats((unsigned char)16);
phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_eta = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
shuffle_mask_eta = spu_rlqwbyte(shuffle_mask_eta, (unsigned int)src1%16);
vector unsigned char tgt_second = spu_rlqwbyte(second_oneup, -((unsigned int)target%16));
vector unsigned char tgt_first = spu_rlqwbyte(oneup, -((unsigned int)target%16));
//alpha: first half of first, second half of second, break at (unsigned int)target%16
src_cmp = spu_splats((unsigned char)((unsigned int)target%16));
gt_res = spu_cmpgt(oneup, src_cmp);
eq_res = spu_cmpeq(oneup, src_cmp);
cmp_res = spu_or(gt_res, eq_res);
phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_alpha = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
//delta: first half of first, first half of second, break at (unsigned int)target%16
vector unsigned char shuffle_mask_delta = spu_shuffle(oneup, tgt_second, (vector unsigned char)shuffle_mask_alpha);
//epsilon: second half of second, second half of first, break at (unsigned int)target%16
vector unsigned char shuffle_mask_epsilon = spu_shuffle(tgt_second, oneup, (vector unsigned char)shuffle_mask_alpha);
//zeta: second half of second, first half of first, break at 16 - (unsigned int)target%16
vector unsigned int shuffle_mask_zeta = spu_rlqwbyte(shuffle_mask_alpha, (unsigned int)target%16);
//beta: first half of first, second half of second, break at num_bytes%16
src_cmp = spu_splats((unsigned char)(num_bytes%16));
gt_res = spu_cmpgt(oneup, src_cmp);
eq_res = spu_cmpeq(oneup, src_cmp);
cmp_res = spu_or(gt_res, eq_res);
phase_change = spu_and(sixteen_uchar, cmp_res);
vector unsigned int shuffle_mask_beta = spu_add((vector unsigned int)phase_change,
(vector unsigned int)oneup);
qword src0_past;
qword src0_present;
qword src1_past;
qword src1_present;
qword tgt_past;
qword tgt_present;
qword in_temp0;
qword in_temp1;
qword out_temp0;
qword out_temp1;
vector signed short sum;
src0_past = si_lqd((qword)address_counter_src0, 0);
src1_past = si_lqd((qword)address_counter_src1, 0);
tgt_past = si_lqd((qword)address_counter_tgt, 0);
for(i = 0; i < num_bytes/16; ++i) {
src0_present = si_lqd((qword)address_counter_src0, 16);
src1_present = si_lqd((qword)address_counter_src1, 16);
tgt_present = si_lqd((qword)address_counter_tgt, 16);
in_temp0 = spu_shuffle(src0_present, src0_past, (vector unsigned char)shuffle_mask_gamma);
in_temp1 = spu_shuffle(src1_present, src1_past, (vector unsigned char)shuffle_mask_eta);
sum = spu_sub((vector signed short)in_temp0, (vector signed short)in_temp1);
out_temp0 = spu_shuffle(tgt_past, (qword)sum, shuffle_mask_delta);
out_temp1 = spu_shuffle(tgt_present, (qword)sum, shuffle_mask_epsilon);
si_stqd(out_temp0, (qword)address_counter_tgt, 0);
si_stqd(out_temp1, (qword)address_counter_tgt, 16);
tgt_past = out_temp1;
src0_past = src0_present;
src1_past = src1_present;
address_counter_src0 = spu_add(address_counter_src0, 16);
address_counter_src1 = spu_add(address_counter_src1, 16);
address_counter_tgt = spu_add(address_counter_tgt, 16);
}
src0_present = si_lqd((qword)address_counter_src0, 16);
src1_present = si_lqd((qword)address_counter_src1, 16);
tgt_present = si_lqd((qword)address_counter_tgt, 16);
in_temp0 = spu_shuffle(src0_present, src0_past, (vector unsigned char) shuffle_mask_gamma);
in_temp1 = spu_shuffle(src1_present, src1_past, (vector unsigned char) shuffle_mask_eta);
sum = spu_sub((vector signed short)in_temp0, (vector signed short)in_temp1);
qword target_temp = spu_shuffle(tgt_present, tgt_past, (vector unsigned char) shuffle_mask_zeta);
qword meld = spu_shuffle((qword)sum, target_temp, (vector unsigned char)shuffle_mask_beta);
out_temp0 = spu_shuffle(tgt_past, meld, shuffle_mask_delta);
out_temp1 = spu_shuffle(tgt_present, meld, shuffle_mask_epsilon);
si_stqd(out_temp0, (qword)address_counter_tgt, 0);
si_stqd(out_temp1, (qword)address_counter_tgt, 16);
return retval;
}
/**
* Sort vertices from top to bottom.
* Compute area and determine front vs. back facing.
* Do coarse clip test against tile bounds
* \return FALSE if tri is totally outside tile, TRUE otherwise
*/
static boolean
setup_sort_vertices(const qword vs)
{
float area, sign;
#if DEBUG_VERTS
if (spu.init.id==0) {
fprintf(stderr, "SPU %u: Triangle:\n", spu.init.id);
print_vertex(v0);
print_vertex(v1);
print_vertex(v2);
}
#endif
{
/* Load the float values for various processing... */
const qword f0 = (qword)(((const struct vertex_header*)si_to_ptr(vs))->data[0]);
const qword f1 = (qword)(((const struct vertex_header*)si_to_ptr(si_rotqbyi(vs, 4)))->data[0]);
const qword f2 = (qword)(((const struct vertex_header*)si_to_ptr(si_rotqbyi(vs, 8)))->data[0]);
/* Check if triangle is completely outside the tile bounds
* Find the min and max x and y positions of the three poits */
const qword minf = min3fq(f0, f1, f2);
const qword maxf = max3fq(f0, f1, f2);
/* Compare min and max against cliprect vals */
const qword maxsmins = si_shufb(maxf, minf, SHUFB4(A,B,a,b));
const qword outside = si_fcgt(maxsmins, si_csflt(setup.cliprect, 0));
/* Use a little magic to work out of the tri is visible or not */
if(si_to_uint(si_xori(si_gb(outside), 0xc))) return FALSE;
/* determine bottom to top order of vertices */
/* A table of shuffle patterns for putting vertex_header pointers into
correct order. Quite magical. */
const qword sort_order_patterns[] = {
SHUFB4(A,B,C,C),
SHUFB4(C,A,B,C),
SHUFB4(A,C,B,C),
SHUFB4(B,C,A,C),
SHUFB4(B,A,C,C),
SHUFB4(C,B,A,C) };
/* Collate y values into two vectors for comparison.
Using only one shuffle constant! ;) */
const qword y_02_ = si_shufb(f0, f2, SHUFB4(0,B,b,C));
const qword y_10_ = si_shufb(f1, f0, SHUFB4(0,B,b,C));
const qword y_012 = si_shufb(y_02_, f1, SHUFB4(0,B,b,C));
const qword y_120 = si_shufb(y_10_, f2, SHUFB4(0,B,b,C));
/* Perform comparison: {y0,y1,y2} > {y1,y2,y0} */
const qword compare = si_fcgt(y_012, y_120);
/* Compress the result of the comparison into 4 bits */
const qword gather = si_gb(compare);
/* Subtract one to attain the index into the LUT. Magical. */
const unsigned int index = si_to_uint(gather) - 1;
/* Load the appropriate pattern and construct the desired vector. */
setup.vertex_headers = si_shufb(vs, vs, sort_order_patterns[index]);
/* Using the result of the comparison, set sign.
Very magical. */
sign = ((si_to_uint(si_cntb(gather)) == 2) ? 1.0f : -1.0f);
}
setup.ebot.ds = spu_sub(setup.vmid->data[0], setup.vmin->data[0]);
setup.emaj.ds = spu_sub(setup.vmax->data[0], setup.vmin->data[0]);
setup.etop.ds = spu_sub(setup.vmax->data[0], setup.vmid->data[0]);
/*
* Compute triangle's area. Use 1/area to compute partial
* derivatives of attributes later.
*/
area = setup.emaj.dx * setup.ebot.dy - setup.ebot.dx * setup.emaj.dy;
setup.oneOverArea = 1.0f / area;
/* The product of area * sign indicates front/back orientation (0/1).
* Just in case someone gets the bright idea of switching the front
* and back constants without noticing that we're assuming their
* values in this operation, also assert that the values are
* what we think they are.
*/
ASSERT(CELL_FACING_FRONT == 0);
ASSERT(CELL_FACING_BACK == 1);
setup.facing = (area * sign > 0.0f)
^ (spu.rasterizer.front_winding == PIPE_WINDING_CW);
return TRUE;
}
inline vector real_t
advec_diff_v(vector real_t cell_size,
vector real_t c2l, vector real_t w2l, vector real_t d2l,
vector real_t c1l, vector real_t w1l, vector real_t d1l,
vector real_t c, vector real_t w, vector real_t d,
vector real_t c1r, vector real_t w1r, vector real_t d1r,
vector real_t c2r, vector real_t w2r, vector real_t d2r)
{
vector real_t acc1, acc2, acc3;
vector real_t wind, diff_term, advec_term;
vector real_t advec_term_pos, advec_term_neg;
vector real_t advec_termR, advec_termL;
acc1 = spu_add(w1l, w);
wind = spu_mul(acc1, HALF);
acc1 = spu_mul(c1l, FIVE);
acc2 = spu_mul(c, TWO);
advec_term_pos = spu_add(acc1, acc2);
advec_term_pos = spu_sub(advec_term_pos, c2l);
acc1 = spu_mul(c1l, TWO);
acc2 = spu_mul(c, FIVE);
advec_term_neg = spu_add(acc1, acc2);
advec_term_neg = spu_sub(advec_term_neg, c1r);
acc1 = (vector real_t)spu_cmpgt(wind, ZERO);
acc1 = spu_and(acc1, advec_term_pos);
acc2 = (vector real_t)spu_cmpgt(ZERO, wind);
acc2 = spu_and(acc2, advec_term_neg);
advec_termL = spu_add(acc1, acc2);
advec_termL = spu_mul(advec_termL, SIXTH);
advec_termL = spu_mul(advec_termL, wind);
acc1 = spu_add(w1r, w);
wind = spu_mul(acc1, HALF);
acc1 = spu_mul(c, FIVE);
acc2 = spu_mul(c1r, TWO);
advec_term_pos = spu_add(acc1, acc2);
advec_term_pos = spu_sub(advec_term_pos, c1l);
acc1 = spu_mul(c, TWO);
acc2 = spu_mul(c1r, FIVE);
advec_term_neg = spu_add(acc1, acc2);
advec_term_neg = spu_sub(advec_term_neg, c2r);
acc1 = (vector real_t)spu_cmpgt(wind, ZERO);
acc1 = spu_and(acc1, advec_term_pos);
acc2 = (vector real_t)spu_cmpgt(ZERO, wind);
acc2 = spu_and(acc2, advec_term_neg);
advec_termR = spu_add(acc1, acc2);
advec_termR = spu_mul(advec_termR, SIXTH);
advec_termR = spu_mul(advec_termR, wind);
acc1 = spu_sub(advec_termL, advec_termR);
advec_term = VEC_DIVIDE(acc1, cell_size);
acc1 = spu_add(d1l, d);
acc1 = spu_mul(acc1, HALF);
acc3 = spu_sub(c1l, c);
acc1 = spu_mul(acc1, acc3);
acc2 = spu_add(d, d1r);
acc2 = spu_mul(acc2, HALF);
acc3 = spu_sub(c, c1r);
acc2 = spu_mul(acc2, acc3);
acc1 = spu_sub(acc1, acc2);
acc2 = spu_mul(cell_size, cell_size);
diff_term = VEC_DIVIDE(acc1, acc2);
return spu_add(advec_term, diff_term);
}
vector double
__divv2df3 (vector double a_in, vector double b_in)
{
/* Variables */
vec_int4 exp, exp_bias;
vec_uint4 no_underflow, overflow;
vec_float4 mant_bf, inv_bf;
vec_ullong2 exp_a, exp_b;
vec_ullong2 a_nan, a_zero, a_inf, a_denorm, a_denorm0;
vec_ullong2 b_nan, b_zero, b_inf, b_denorm, b_denorm0;
vec_ullong2 nan;
vec_uint4 a_exp, b_exp;
vec_ullong2 a_mant_0, b_mant_0;
vec_ullong2 a_exp_1s, b_exp_1s;
vec_ullong2 sign_exp_mask;
vec_double2 a, b;
vec_double2 mant_a, mant_b, inv_b, q0, q1, q2, mult;
/* Constants */
vec_uint4 exp_mask_u32 = spu_splats((unsigned int)0x7FF00000);
vec_uchar16 splat_hi = (vec_uchar16) {
0,1,2,3, 0,1,2,3, 8, 9,10,11, 8,9,10,11
};
vec_uchar16 swap_32 = (vec_uchar16) {
4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11
};
vec_ullong2 exp_mask = spu_splats(0x7FF0000000000000ULL);
vec_ullong2 sign_mask = spu_splats(0x8000000000000000ULL);
vec_float4 onef = spu_splats(1.0f);
vec_double2 one = spu_splats(1.0);
vec_double2 exp_53 = (vec_double2)spu_splats(0x0350000000000000ULL);
sign_exp_mask = spu_or(sign_mask, exp_mask);
/* Extract the floating point components from each of the operands including
* exponent and mantissa.
*/
a_exp = (vec_uint4)spu_and((vec_uint4)a_in, exp_mask_u32);
a_exp = spu_shuffle(a_exp, a_exp, splat_hi);
b_exp = (vec_uint4)spu_and((vec_uint4)b_in, exp_mask_u32);
b_exp = spu_shuffle(b_exp, b_exp, splat_hi);
a_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)a_in, sign_exp_mask), 0);
a_mant_0 = spu_and(a_mant_0, spu_shuffle(a_mant_0, a_mant_0, swap_32));
b_mant_0 = (vec_ullong2)spu_cmpeq((vec_uint4)spu_andc((vec_ullong2)b_in, sign_exp_mask), 0);
b_mant_0 = spu_and(b_mant_0, spu_shuffle(b_mant_0, b_mant_0, swap_32));
a_exp_1s = (vec_ullong2)spu_cmpeq(a_exp, exp_mask_u32);
b_exp_1s = (vec_ullong2)spu_cmpeq(b_exp, exp_mask_u32);
/* Identify all possible special values that must be accommodated including:
* +-denorm, +-0, +-infinity, and NaNs.
*/
a_denorm0= (vec_ullong2)spu_cmpeq(a_exp, 0);
a_nan = spu_andc(a_exp_1s, a_mant_0);
a_zero = spu_and (a_denorm0, a_mant_0);
a_inf = spu_and (a_exp_1s, a_mant_0);
a_denorm = spu_andc(a_denorm0, a_zero);
b_denorm0= (vec_ullong2)spu_cmpeq(b_exp, 0);
b_nan = spu_andc(b_exp_1s, b_mant_0);
b_zero = spu_and (b_denorm0, b_mant_0);
b_inf = spu_and (b_exp_1s, b_mant_0);
b_denorm = spu_andc(b_denorm0, b_zero);
/* Scale denorm inputs to into normalized numbers by conditionally scaling the
* input parameters.
*/
a = spu_sub(spu_or(a_in, exp_53), spu_sel(exp_53, a_in, sign_mask));
a = spu_sel(a_in, a, a_denorm);
b = spu_sub(spu_or(b_in, exp_53), spu_sel(exp_53, b_in, sign_mask));
b = spu_sel(b_in, b, b_denorm);
/* Extract the divisor and dividend exponent and force parameters into the signed
* range [1.0,2.0) or [-1.0,2.0).
*/
exp_a = spu_and((vec_ullong2)a, exp_mask);
exp_b = spu_and((vec_ullong2)b, exp_mask);
mant_a = spu_sel(a, one, (vec_ullong2)exp_mask);
mant_b = spu_sel(b, one, (vec_ullong2)exp_mask);
/* Approximate the single reciprocal of b by using
* the single precision reciprocal estimate followed by one
* single precision iteration of Newton-Raphson.
*/
mant_bf = spu_roundtf(mant_b);
inv_bf = spu_re(mant_bf);
inv_bf = spu_madd(spu_nmsub(mant_bf, inv_bf, onef), inv_bf, inv_bf);
/* Perform 2 more Newton-Raphson iterations in double precision. The
* result (q1) is in the range (0.5, 2.0).
*/
inv_b = spu_extend(inv_bf);
inv_b = spu_madd(spu_nmsub(mant_b, inv_b, one), inv_b, inv_b);
q0 = spu_mul(mant_a, inv_b);
q1 = spu_madd(spu_nmsub(mant_b, q0, mant_a), inv_b, q0);
/* Determine the exponent correction factor that must be applied
* to q1 by taking into account the exponent of the normalized inputs
* and the scale factors that were applied to normalize them.
*/
exp = spu_rlmaska(spu_sub((vec_int4)exp_a, (vec_int4)exp_b), -20);
exp = spu_add(exp, (vec_int4)spu_add(spu_and((vec_int4)a_denorm, -0x34), spu_and((vec_int4)b_denorm, 0x34)));
/* Bias the quotient exponent depending on the sign of the exponent correction
* factor so that a single multiplier will ensure the entire double precision
* domain (including denorms) can be achieved.
*
* exp bias q1 adjust exp
* ===== ======== ==========
* positive 2^+65 -65
* negative 2^-64 +64
*/
exp_bias = spu_xor(spu_rlmaska(exp, -31), 64);
exp = spu_sub(exp, exp_bias);
q1 = spu_sel(q1, (vec_double2)spu_add((vec_int4)q1, spu_sl(exp_bias, 20)), exp_mask);
/* Compute a multiplier (mult) to applied to the quotient (q1) to produce the
* expected result. On overflow, clamp the multiplier to the maximum non-infinite
* number in case the rounding mode is not round-to-nearest.
*/
exp = spu_add(exp, 0x3FF);
no_underflow = spu_cmpgt(exp, 0);
overflow = spu_cmpgt(exp, 0x7FE);
exp = spu_and(spu_sl(exp, 20), (vec_int4)no_underflow);
exp = spu_and(exp, (vec_int4)exp_mask);
mult = spu_sel((vec_double2)exp, (vec_double2)(spu_add((vec_uint4)exp_mask, -1)), (vec_ullong2)overflow);
/* Handle special value conditions. These include:
*
* 1) IF either operand is a NaN OR both operands are 0 or INFINITY THEN a NaN
* results.
* 2) ELSE IF the dividend is an INFINITY OR the divisor is 0 THEN a INFINITY results.
* 3) ELSE IF the dividend is 0 OR the divisor is INFINITY THEN a 0 results.
*/
mult = spu_andc(mult, (vec_double2)spu_or(a_zero, b_inf));
mult = spu_sel(mult, (vec_double2)exp_mask, spu_or(a_inf, b_zero));
nan = spu_or(a_nan, b_nan);
nan = spu_or(nan, spu_and(a_zero, b_zero));
nan = spu_or(nan, spu_and(a_inf, b_inf));
mult = spu_or(mult, (vec_double2)nan);
/* Scale the final quotient */
q2 = spu_mul(q1, mult);
return (q2);
}
/* Scans the string pointed to by s for the character c and
* returns a pointer to the last occurance of c. If
* c is not found, then NULL is returned.
*/
char * strrchr(const char *s, int c)
{
int nskip;
vec_uchar16 *ptr, data, vc;
vec_uint4 cmp_c, cmp_0, cmp;
vec_uint4 res_ptr, res_cmp;
vec_uint4 mask, result;
vec_uint4 one = spu_splats(0xffffU);
/* Scan memory array a quadword at a time. Skip leading
* mis-aligned bytes.
*/
ptr = (vec_uchar16 *)s;
nskip = -((unsigned int)(ptr) & 15);
mask = spu_rlmask(one, nskip);
vc = spu_splats((unsigned char)(c));
data = *ptr++;
ptr = (vec_uchar16 *)((unsigned int)ptr & ~15);
cmp_c = spu_and(spu_gather(spu_cmpeq(data, vc)), mask);
cmp_0 = spu_and(spu_gather(spu_cmpeq(data, 0)), mask);
res_ptr = spu_splats(0U);
res_cmp = spu_splats(0U);
while (spu_extract(cmp_0, 0) == 0) {
cmp = spu_cmpeq(cmp_c, 0);
res_ptr = spu_sel(spu_promote((unsigned int)(ptr), 0), res_ptr, cmp);
res_cmp = spu_sel(cmp_c, res_cmp, cmp);
data = *ptr++;
cmp_c = spu_gather(spu_cmpeq(data, vc));
cmp_0 = spu_gather(spu_cmpeq(data, 0));
cmp = spu_cmpeq(cmp_c, 0);
}
/* Compute the location of the last character before termination
* character.
*
* First mask off compare results following the first termination character.
*/
mask = spu_sl(one, 31 - spu_extract(spu_cntlz(cmp_0), 0));
cmp_c = spu_and(cmp_c, mask);
/* Conditionally update res_ptr and res_cmd if a match was found in the last
* quadword.
*/
cmp = spu_cmpeq(cmp_c, 0);
res_ptr = spu_sel(spu_promote((unsigned int)(ptr), 0), res_ptr, cmp);
res_cmp = spu_sel(cmp_c, res_cmp, cmp);
/* Bit reserve res_cmp for locating last occurance.
*/
mask = spu_cmpeq(res_cmp, 0);
res_cmp = (vec_uint4)spu_maskb(spu_extract(res_cmp, 0));
res_cmp = spu_gather((vec_uchar16)spu_shuffle(res_cmp, res_cmp,
VEC_LITERAL(vec_uchar16,
15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)));
/* Compute the location (ptr) of the last occurance of c. If no
* occurance was found (ie, element 0 of res_cmp == 0, then return
* NULL.
*/
result = spu_sub(spu_add(res_ptr, 15), spu_cntlz(res_cmp));
result = spu_andc(result, mask);
return ((char *)spu_extract(result, 0));
}
void process_render_tasks(unsigned long eah_render_tasks, unsigned long eal_render_tasks)
{
const vec_uchar16 SHUFFLE_MERGE_BYTES = (vec_uchar16) { // merge lo bytes from unsigned shorts (array)
1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31
};
const vec_uchar16 SHUFFLE_GET_BUSY_WITH_ONES = (vec_uchar16) { // get busy flag with ones in unused bytes
0xc0, 0xc0, 2, 3, 0xc0,0xc0,0xc0,0xc0, 0xc0,0xc0,0xc0,0xc0
};
const vec_uchar16 ZERO_BYTES = (vec_uchar16) spu_splats(0);
char trianglebuffer[ 256 + TRIANGLE_MAX_SIZE ];
char sync_buffer[128+127];
void* aligned_sync_buffer = (void*) ( ((unsigned long)sync_buffer+127) & ~127 );
RenderableCacheLine* cache = (RenderableCacheLine*) aligned_sync_buffer;
unsigned long long cache_ea;
spu_mfcdma64(&cache_ea, eah_render_tasks, eal_render_tasks, sizeof(cache_ea), 0, MFC_GET_CMD);
mfc_write_tag_mask(1<<0);
mfc_read_tag_status_all();
while (cache_ea) {
// terminate immediately if possible
if (spu_stat_in_mbox())
return;
// read the cache line
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_GETLLAR_CMD);
spu_readch(MFC_RdAtomicStat);
unsigned int endTriangle = cache->endTriangle;
vec_ushort8 testTriangle = spu_splats((unsigned short) endTriangle);
// first look for short chunks
vec_uchar16 next = cache->chunkNext;
vec_uchar16 nextmask = spu_and(next, spu_splats((unsigned char)CHUNKNEXT_MASK));
// change next to word offset, note we don't care what the low bit shifted in is
vec_uchar16 firstshuf = (vec_uchar16) spu_sl( (vec_ushort8)nextmask, 1 );
vec_uchar16 trishufhi = spu_or ( firstshuf, spu_splats((unsigned char) 1));
vec_uchar16 trishuflo = spu_and( firstshuf, spu_splats((unsigned char) 254));
vec_ushort8 start0 = cache->chunkStart[0];
vec_ushort8 start1 = cache->chunkStart[1];
vec_ushort8 nstart0 = spu_shuffle( start0, start1, spu_shuffle( trishuflo, trishufhi, SHUF0 ) );
vec_ushort8 nstart1 = spu_shuffle( start0, start1, spu_shuffle( trishuflo, trishufhi, SHUF1 ) );
vec_ushort8 starteq0 = spu_cmpeq( nstart0, spu_splats((unsigned short)0) );
vec_ushort8 starteq1 = spu_cmpeq( nstart1, spu_splats((unsigned short)0) );
vec_ushort8 end0 = spu_sel( nstart0, spu_splats((unsigned short)4096), starteq0);
vec_ushort8 end1 = spu_sel( nstart1, spu_splats((unsigned short)4096), starteq1);
vec_ushort8 len0 = spu_sub( end0, start0);
vec_ushort8 len1 = spu_sub( end1, start1);
vec_ushort8 small0 = spu_cmpgt( spu_splats((unsigned short)17), len0);
vec_ushort8 small1 = spu_cmpgt( spu_splats((unsigned short)17), len1);
vec_uchar16 small = (vec_uchar16) spu_shuffle( small0, small1, MERGE );
vec_uint4 smallChunkGather = spu_gather(small);
// check to see if chunk is already at the last triangle
vec_uint4 doneChunkGather = spu_gather( (vec_uchar16) spu_shuffle(
(vec_uchar16) spu_cmpeq(testTriangle, cache->chunkTriangle[0]),
(vec_uchar16) spu_cmpeq(testTriangle, cache->chunkTriangle[1]),
SHUFFLE_MERGE_BYTES) );
// check if the chunk is free
vec_uint4 freeChunkGather = spu_gather(
spu_cmpeq( spu_splats( (unsigned char) CHUNKNEXT_FREE_BLOCK ), cache->chunkNext ) );
// check to see if the chunk is being processed
vec_uint4 busyChunkGather = spu_gather(
spu_cmpgt( cache->chunkNext, //spu_and(cache->chunkNext, CHUNKNEXT_MASK),
spu_splats( (unsigned char) (CHUNKNEXT_BUSY_BIT-1) ) ) );
// doneChunkGather, freeChunkGather, busyChunkGather - rightmost 16 bits of word 0
// note that if freeChunkGather is true then busyChunkGather must also be true
// done=false, free=false, busy=false -> can process
// free=false, busy=false -> can be merged
// decide which chunk to process
vec_uint4 mayProcessGather = spu_nor( doneChunkGather, busyChunkGather );
vec_uint4 mayProcessShortGather = spu_and( mayProcessGather, smallChunkGather );
vec_uint4 shortSelMask = spu_cmpeq( mayProcessShortGather, spu_splats(0U) );
vec_uint4 mayProcessSelection = spu_sel( mayProcessShortGather, mayProcessGather, shortSelMask );
/*
if (!spu_extract(shortSelMask, 0))
printf("taken short: may=%04x short=%04x mayshort=%04x mask=%04x sel=%04x\n",
spu_extract(mayProcessGather, 0) & 0xffff,
spu_extract(smallChunkGather, 0),
spu_extract(mayProcessShortGather, 0),
spu_extract(shortSelMask, 0) & 0xffff,
spu_extract(mayProcessSelection, 0) & 0xffff );
*/
vec_uint4 mayProcessBits = spu_sl( mayProcessSelection, 16);
unsigned int chunkToProcess = spu_extract( spu_cntlz( mayProcessBits ), 0);
unsigned int freeChunk = spu_extract( spu_cntlz( spu_sl( freeChunkGather, 16 ) ), 0);
// if there's nothing to process, try the next cache line in the rendering tasks list
if (!spu_extract(mayProcessBits, 0)) {
trynextcacheline:
cache_ea = cache->next;
// sleep();
continue;
}
unsigned int chunkStart = cache->chunkStartArray [chunkToProcess];
unsigned int chunkTriangle = cache->chunkTriangleArray[chunkToProcess];
unsigned int chunkNext = cache->chunkNextArray [chunkToProcess] & CHUNKNEXT_MASK;
unsigned int chunkEnd = (cache->chunkStartArray [chunkNext]-1) & (NUMBER_OF_TILES-1);
unsigned int chunkLength = 1 + chunkEnd-chunkStart;
// only need an extra block if the block is especially long
if (chunkLength <= NUMBER_OF_TILES_PER_CHUNK) {
freeChunk = 32;
}
// mark this block as busy
cache->chunkNextArray[chunkToProcess] |= CHUNKNEXT_BUSY_BIT;
// if there's at least one free chunk, claim it
if (freeChunk != 32) {
cache->chunkNextArray[freeChunk] = CHUNKNEXT_RESERVED;
cache->chunkTriangleArray[freeChunk] = chunkTriangle;
}
// write the cache line back
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_PUTLLC_CMD);
if (spu_readch(MFC_RdAtomicStat) & MFC_PUTLLC_STATUS)
continue;
#ifdef INFO
printf("[%d] Claimed chunk %d (%d-%d len %d) at tri %x end %x with free chunk %d\n", _SPUID,
chunkToProcess, chunkStart, chunkEnd, chunkLength, chunkTriangle, endTriangle,
freeChunk!=32 ? freeChunk : -1 );
// debug_render_tasks(cache);
#endif
Triangle* triangle;
int firstTile;
do {
// read the triangle data for the current triangle
unsigned int extra = chunkTriangle & 127;
unsigned long long trianglebuffer_ea = cache_ea + TRIANGLE_OFFSET_FROM_CACHE_LINE + (chunkTriangle & ~127);
triangle = (Triangle*) (trianglebuffer+extra);
unsigned int length = (extra + TRIANGLE_MAX_SIZE + 127) & ~127;
// ensure DMA slot available
do {} while (!spu_readchcnt(MFC_Cmd));
spu_mfcdma64(trianglebuffer, mfc_ea2h(trianglebuffer_ea), mfc_ea2l(trianglebuffer_ea),
length, 0, MFC_GET_CMD);
mfc_write_tag_mask(1<<0);
mfc_read_tag_status_all();
// get the triangle deltas
firstTile = findFirstTriangleTile(triangle, chunkStart, chunkEnd);
if (firstTile>=0)
break;
// no match, try next triangle
chunkTriangle = triangle->next_triangle;
} while (chunkTriangle != endTriangle);
// if we actually have something to process...
if (firstTile>=0) {
// the "normal" splitting will now become:
// chunkStart .. (firstTile-1) -> triangle->next_triangle
// firstTile .. (firstTile+NUM-1) -> chunkTriangle (BUSY)
// (firstTile+NUM) .. chunkEnd -> chunkTriangle (FREE)
int tailChunk;
int thisChunk;
int nextBlockStart;
int thisBlockStart;
int realBlockStart;
do {
retry:
// read the cache line
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_GETLLAR_CMD);
spu_readch(MFC_RdAtomicStat);
// calculate start of next block
nextBlockStart = firstTile + NUMBER_OF_TILES_PER_CHUNK;
if (nextBlockStart > chunkEnd)
nextBlockStart = chunkEnd+1;
// calculate start of block to mark as busy
thisBlockStart = nextBlockStart - NUMBER_OF_TILES_PER_CHUNK;
if (thisBlockStart < chunkStart)
thisBlockStart = chunkStart;
realBlockStart = thisBlockStart;
#ifdef INFO
printf("[%d] nextBlockStart=%d, realBlockStart=%d, thisBlockStart=%d, chunkStart=%d\n", _SPUID,
nextBlockStart, realBlockStart, thisBlockStart, chunkStart);
#endif
// allocate some more free chunks
vec_uint4 freeChunkGather2 = spu_sl(spu_gather(spu_cmpeq(
spu_splats((unsigned char)CHUNKNEXT_FREE_BLOCK), cache->chunkNext)), 16);
unsigned int freeChunk2 = spu_extract(spu_cntlz(freeChunkGather2), 0);
if (freeChunk == 32) {
// if we didn't have one before, try again
freeChunk = freeChunk2;
// and try to get the second one
freeChunkGather2 = spu_andc( freeChunkGather2, spu_promote(0x80000000>>freeChunk2, 0) );
freeChunk2 = spu_extract(spu_cntlz(freeChunkGather2), 0);
} else {
// speculatively clear the free chunk just in case we don't need it
cache->chunkNextArray[freeChunk] = CHUNKNEXT_FREE_BLOCK;
}
#ifdef INFO
printf("[%d] Free chunks %d and %d, cN=%d, nBS=%d, cE=%d, tBS=%d, cS=%d\n",
_SPUID, freeChunk, freeChunk2, chunkNext, nextBlockStart, chunkEnd, thisBlockStart, chunkStart );
#endif
// mark region after as available for processing if required
if (nextBlockStart < chunkEnd) {
if (freeChunk==32) {
// if no free chunk, relinquish entire block and write back
cache->chunkNextArray[chunkToProcess] = chunkNext;
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_PUTLLC_CMD);
// if writeback failed, we *might* have a free block, retry
if (spu_readch(MFC_RdAtomicStat) & MFC_PUTLLC_STATUS)
goto retry;
// otherwise give up and try the next cache line
goto trynextcacheline;
}
cache->chunkStartArray[freeChunk] = nextBlockStart;
cache->chunkNextArray[freeChunk] = chunkNext;
cache->chunkTriangleArray[freeChunk] = chunkTriangle;
cache->chunkNextArray[chunkToProcess] = freeChunk | CHUNKNEXT_BUSY_BIT;
tailChunk = freeChunk;
#ifdef INFO
printf("[%d] Insert tail, tailChunk=%d, chunkNext=%d, chunkToProcess=%d\n", _SPUID, tailChunk, chunkNext, chunkToProcess);
debug_render_tasks(cache);
#endif
} else {
// we're gonna use freeChunk2 for the "in front" block, as we've not
// used freeChunk, let's use it as it's more likely to have a free chunk
freeChunk2 = freeChunk;
tailChunk = chunkNext;
}
// mark region before as available if required and possible
thisChunk = chunkToProcess;
if (thisBlockStart > chunkStart) {
if (freeChunk2 != 32) {
// mark this region as busy
cache->chunkStartArray[freeChunk2]=thisBlockStart;
cache->chunkNextArray[freeChunk2]=tailChunk | CHUNKNEXT_BUSY_BIT;
cache->chunkTriangleArray[freeChunk2]=chunkTriangle;
// mark region before as available for processing
cache->chunkNextArray[chunkToProcess]=freeChunk2;
cache->chunkTriangleArray[chunkToProcess]=triangle->next_triangle;
thisChunk = freeChunk2;
#ifdef INFO
printf("[%d] Insert new head, tailChunk=%d, chunkNext=%d, thisChunk=%d\n", _SPUID, tailChunk, chunkNext, thisChunk);
debug_render_tasks(cache);
#endif
} else {
// need to keep whole block, update info and mark bust
cache->chunkTriangleArray[chunkToProcess]=chunkTriangle;
cache->chunkNextArray[chunkToProcess]=tailChunk | CHUNKNEXT_BUSY_BIT;
realBlockStart = chunkStart;
printf("[%d] Keep whole block, tailChunk=%d, chunkNext=%d, thisChunk=%d\n", _SPUID, tailChunk, chunkNext, thisChunk);
debug_render_tasks(cache);
#ifdef INFO
#endif
sleep();
}
}
// merge chunks
merge_cache_blocks(cache);
// write the cache line back
spu_mfcdma64(cache, mfc_ea2h(cache_ea), mfc_ea2l(cache_ea), 128, 0, MFC_PUTLLC_CMD);
} while (spu_readch(MFC_RdAtomicStat) & MFC_PUTLLC_STATUS);
// finally after the write succeeded, update the variables
chunkNext = tailChunk;
chunkToProcess = thisChunk;
chunkStart = firstTile; //thisBlockStart;
chunkLength = nextBlockStart - firstTile;
chunkEnd = chunkStart + chunkLength - 1;
freeChunk = 32;
// now we can process the block up to endTriangle
initTileBuffers(thisBlockStart, chunkEnd);
int ok=0;
while (chunkTriangle != endTriangle) {
#ifdef INFO
printf("[%d] Processing chunk %d at %4d len %4d, triangle %04x first=%d tbs=%d\n",
_SPUID, chunkToProcess, chunkStart, chunkLength,
chunkTriangle, firstTile, thisBlockStart);
#endif
// and actually process that triangle on these chunks
processTriangleChunks(triangle, cache, thisBlockStart, chunkEnd, chunkTriangle, ok);
ok=1;
#ifdef PAUSE
sleep();
#endif
// and advance to the next-triangle
chunkTriangle = triangle->next_triangle;
// this should only ever happen if we're running really low on cache line slots
// basically, if we pick up a block with more than NUMBER_OF_TILES_PER_CHUNK and
// there's no slot to store the pre-NUMBER_OF_TILES_PER_CHUNK tiles.
// in this case, we process from thisBlockStart only (because we know that from
// chunkStart to there has no result) and then we only process one triangle
if (chunkStart != realBlockStart) {
/*
printf("[%d] chunkStart=%d != realBlockStart %d, chunkEnd=%d, "
"firstTile=%d chunk=%d\n",
_SPUID, chunkStart, realBlockStart, chunkEnd,
firstTile, chunkToProcess);
debug_render_tasks(cache);
*/
// abort the while loop
break;
}
// read the next triangle
unsigned int extra = chunkTriangle & 127;
unsigned long long trianglebuffer_ea = cache_ea + TRIANGLE_OFFSET_FROM_CACHE_LINE + (chunkTriangle & ~127);
triangle = (Triangle*) (trianglebuffer+extra);
unsigned int length = (extra + TRIANGLE_MAX_SIZE + 127) & ~127;
// ensure DMA slot available
do {} while (!spu_readchcnt(MFC_Cmd));
spu_mfcdma64(trianglebuffer, mfc_ea2h(trianglebuffer_ea),
mfc_ea2l(trianglebuffer_ea), length, 0, MFC_GET_CMD);
mfc_write_tag_mask(1<<0);
mfc_read_tag_status_all();
} // until chunkTriangle == endTriangle
// flush any output buffers
flushTileBuffers(thisBlockStart, chunkEnd);
} // firstTile>=0
void MinMaxBinFindBest3SIMD(minmaxbin_t *mmb, kdbuffer_t *result)
{
int i;
for(i=1; i < mmb->numbins; i++)
{
int j = mmb->numbins - i - 1;
vector float *min = (vector float *)mmb->minbins[i].b;
vector float *max = (vector float *)mmb->maxbins[j].b;
min[0] = spu_add(min[0], min[-1]);
max[0] = spu_add(max[0], max[1]);
}
vector float *vmax = (vector float*)result->baabb.max;
vector float *vmin = (vector float*)result->baabb.min;
vector float vwidth = spu_abs( spu_sub(*vmax, *vmin) );
vector float vnumbins = spu_splats(1/(float)mmb->numbins);
vector float vdelta = spu_mul(vwidth, vnumbins);
vector float vx = spu_add(*vmin, vdelta);
vector float vside = { vwidth[1] * vwidth[2], vwidth[0] * vwidth[2], vwidth[0] * vwidth[1], 0 };
vector float invarea = spu_splats( 1/(vwidth[0] * vside[0]));
vector float vctravers = spu_splats(2.0f);
vector float vbestcost = spu_splats(mmb->bestcost);
vector int vbesti = spu_splats(0);
vector float vbestx = vx;
for(i=0; i < mmb->numbins-1; i++)
{
vector float aleft, aright;
AreaLeftRight(*vmin, *vmax, vside, vx, &aleft, &aright);
vector float *vminbin = (vector float *)mmb->minbins[i].b;
vector float *vmaxbin = (vector float *)mmb->maxbins[i+1].b;
vector float cost = SAHCostSIMD(invarea, vctravers, *vminbin, aleft, *vmaxbin, aright);
vector unsigned int cmp = spu_cmpgt(cost, vbestcost);
vbestcost = spu_sel(cost, vbestcost, cmp);
vbesti = spu_sel(spu_splats(i), vbesti, cmp);
vbestx = spu_sel(vx, vbestx, cmp);
vx = spu_add(vx, vdelta);
}
int axis = 0;
float bestcost = vbestcost[axis];
if(vbestcost[1] < bestcost)
{
axis = 1;
bestcost = vbestcost[1];
}
if(vbestcost[2] < bestcost)
{
axis = 2;
bestcost = vbestcost[2];
}
int index = vbesti[axis];
result->plane = vbestx[axis];
result->axis = axis;
result->left_size = (int)mmb->minbins[ index ].b[axis];
result->right_size = (int)mmb->maxbins[ index+1 ].b[axis];
mmb->bestcost = vbestcost[axis];
}
void _compute( unsigned int bs, unsigned int k, unsigned int row, vector float *BKJ, float *B, float *A )
{
vector float *BIJ, aik;
unsigned int i;
// BIJ = (vector float*)B;
// pfv(BIJ);
for( ; k < 64 ; k++ )
{
aik = spu_splats( A[k*64+row] );
BIJ = (vector float*)( B + ( k * 64 ) );
BIJ[0] = spu_sub( BIJ[0], spu_mul( BKJ[0], aik ) );
BIJ[1] = spu_sub( BIJ[1], spu_mul( BKJ[1], aik ) );
BIJ[2] = spu_sub( BIJ[2], spu_mul( BKJ[2], aik ) );
BIJ[3] = spu_sub( BIJ[3], spu_mul( BKJ[3], aik ) );
BIJ[4] = spu_sub( BIJ[4], spu_mul( BKJ[4], aik ) );
BIJ[5] = spu_sub( BIJ[5], spu_mul( BKJ[5], aik ) );
BIJ[6] = spu_sub( BIJ[6], spu_mul( BKJ[6], aik ) );
BIJ[7] = spu_sub( BIJ[7], spu_mul( BKJ[7], aik ) );
BIJ[8] = spu_sub( BIJ[8], spu_mul( BKJ[8], aik ) );
BIJ[9] = spu_sub( BIJ[9], spu_mul( BKJ[9], aik ) );
BIJ[10] = spu_sub( BIJ[10], spu_mul( BKJ[10], aik ) );
BIJ[11] = spu_sub( BIJ[11], spu_mul( BKJ[11], aik ) );
BIJ[12] = spu_sub( BIJ[12], spu_mul( BKJ[12], aik ) );
BIJ[13] = spu_sub( BIJ[13], spu_mul( BKJ[13], aik ) );
BIJ[14] = spu_sub( BIJ[14], spu_mul( BKJ[14], aik ) );
BIJ[15] = spu_sub( BIJ[15], spu_mul( BKJ[15], aik ) );
}
}
