/*
* Call completion cb functions:
* Take care of case where we allocated temp buf
*/
static void
mic_dma_lib_interrupt_handler(struct dma_channel *chan)
{
int i = 0;
int ring_size = chan->intr_ring.ring.size;
struct dma_completion_cb **temp = chan->intr_ring.comp_cb_array;
struct dma_completion_cb *cb;
int new_tail, old_tail;
if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC &&
mic_hw_stepping(chan->dma_ctx->device_num) >= KNC_B0_STEP) {
unsigned long error = *((uint32_t*)chan->chan->dstat_wb_loc);
if (unlikely(test_bit(31, &error)))
printk(KERN_ERR "DMA h/w error - %s %d, dstatwb=%lx\n",
__func__, __LINE__, error);
}
new_tail = read_tail(&chan->intr_ring.ring);
old_tail = chan->intr_ring.old_tail;
for (; i < ring_size && old_tail != new_tail;
old_tail = incr_rb_index(old_tail, ring_size), i++) {
cb = (struct dma_completion_cb *)xchg(&temp[old_tail], NULL);
if (cb) {
cb->dma_completion_func(cb->cb_cookie);
}
}
chan->intr_ring.old_tail = new_tail;
update_tail(&chan->intr_ring.ring, new_tail);
wake_up(&chan->intr_wq);
if (i == ring_size && old_tail != new_tail) {
printk(KERN_ERR PR_PREFIX "Something went wrong, old tail = %d, new tail = %d\n",
old_tail, new_tail);
}
}
//write the next available entry on log
//update lastblock and lastentry
//return 0 on success
//return -1 on log full
int StorageLog::update_log(uint32_t method, u64 node1, u64 node2) {
if(lastblock >= MAXBLOCKNUM) {
printf("maximum block num reached in update_log\n");
return -1;
}
uint32_t generation;
unsigned char* superblock_pt;
ssize_t freeresult;
superblock_pt = get_superblock(fd);
Superblock* pt = (Superblock*) superblock_pt;
if(isValid(superblock_pt)) {
generation = pt->generation;
}
else {
printf("superblock is invalid in update_log\n");
exit(-1);
}
freeresult = munmap(superblock_pt, PAGESIZE);
if(freeresult < 0) {
printf("free superblock failed in update_logp\n");
}
Requestentry entry;
entry.method = method;
entry.node1 = node1;
entry.node2 = node2;
printf("Before updating log entry: \n");
update_log_entry(fd, lastblock, lastentry, entry, generation);
update_tail(lastblock, lastentry);
return 0;
}
void cp_buffer(int side){
int avail_out = num_free_in_buffer(OUT);
int avail_side = num_in_buffer(side);
int max = avail_out < avail_side ? avail_out : avail_side;
vector signed int *out_head;
if(mcb[am].local[OUT] < 255)
out_head = (vector signed int*) &md[ mcb[am].local[OUT] ].idx[ (mcb[am].id+1)&1 ][HEAD];
else
out_head = (vector signed int*) &md[am].idx[OUT][HEAD];
vector unsigned int cmp_v;
vector signed int from_size = spu_splats( mcb[am].buffer_size[side] );
vector signed int out_size = spu_splats( mcb[ mcb[am].local[OUT] ].buffer_size[ (mcb[am].id+1)&1 ] );
vector signed int ones = {1,1,1,1};
vector signed int zeros = {0,0,0,0};
int i;
for(i = 0; i < max; i++){
md[am].buffer[OUT][spu_extract( *out_head,0)] = md[am].buffer[side][spu_extract(md[am].idx[side][TAIL],0)];
// update idx
md[am].idx[side][TAIL] = spu_add(md[am].idx[side][TAIL], ones);
cmp_v = spu_cmpeq(md[am].idx[side][TAIL],from_size);
md[am].idx[side][TAIL] = spu_sel(md[am].idx[side][TAIL], zeros, cmp_v);
*out_head = spu_add(*out_head,ones);
cmp_v = spu_cmpeq(*out_head, out_size);
*out_head = spu_sel(*out_head,zeros,cmp_v);
}
update_tail(side);
md[am].consumed[side] += max;
if(mcb[am].local[OUT] < 255 && md[am].consumed[side] == mcb[am].data_size[side]){
md[am].depleted[side] = 1;
md[am].done = 1;
--num_active_mergers;
}
}
void merge_buffers(){
vector unsigned int cmp_v, cmp_v2;
const vector signed int one_at_0 = {1,0,0,0};
const vector signed int one_at_1 = {0,1,0,0};
const vector signed int one_at_2 = {0,0,1,0};
const vector signed int ones = {1,1,1,1};
const vector signed int zeros = {0,0,0,0};
const vector unsigned char cmp_v_shuffle_mask = {31,31,31,31,
31,31,31,31,
31,31,31,31,
31,31,31,31};
vector unsigned char rev_mask;
const vector unsigned char rev_left = {12,13,14,15,
8,9,10,11,
4,5,6,7,
0,1,2,3};
const vector unsigned char rev_right = {28,29,30,31,
24,25,26,27,
20,21,22,23,
16,17,18,19};
vector signed int *out_head_idx;
if(mcb[am].local[OUT] < 255){
int parent_idx = mcb[am].local[OUT];
int side = (mcb[am].id+1)&1;
out_head_idx = (vector signed int*) &md[parent_idx].idx[side][HEAD];
} else {
out_head_idx = (vector signed int*) &md[am].idx[OUT][HEAD];
}
vector signed int *left_tail_idx = (vector signed int*) &md[am].idx[LEFT][TAIL];
vector signed int *right_tail_idx = (vector signed int*) &md[am].idx[RIGHT][TAIL];
vector signed int size_v = {mcb[am].buffer_size[LEFT], mcb[am].buffer_size[RIGHT], mcb[am].buffer_size[OUT], 0};
vector signed int avail_v = {num_in_buffer(LEFT), num_in_buffer(RIGHT), num_free_in_buffer(OUT), 1};
vector signed int avail_before = { spu_extract(avail_v, 0), spu_extract(avail_v, 1), 0, 0 };
vector unsigned int avail = spu_gather( spu_cmpgt(avail_v, zeros) ); // avail = 0x0F if all avail_v > zeros
vector signed int *left, *right, *out;
left = (vector signed int*) &md[am].buffer[LEFT][ spu_extract(*left_tail_idx,0) ];
right = (vector signed int*) &md[am].buffer[RIGHT][ spu_extract(*right_tail_idx,0) ];
out = (vector signed int*) &md[am].buffer[OUT][ spu_extract(*out_head_idx,0) ];
#ifdef TRACE_TIME
dec_val2 = spu_read_decrementer();
#endif
while(spu_extract(avail,0) == 0x0F){
// cmp left and right to determine who gets eaten
cmp_v = spu_cmpgt(*left,*right);
cmp_v = spu_shuffle(cmp_v, cmp_v, cmp_v_shuffle_mask);
// cmp_v = {FFFF,FFFF,FFFF,FFFF} if left[3] > right[3]
*out = spu_sel(*left,*right,cmp_v);
rev_mask = spu_sel(rev_right,rev_left,(vector unsigned char)cmp_v);
*left = spu_shuffle(*left,*right,rev_mask);
// data to be sorted is now in out and left, left in descending order
sort_vectors(out,left);
// update index of the used side
if( spu_extract(cmp_v,0) ){
// left[3] > right[3]
*right_tail_idx = spu_add(*right_tail_idx,ones);
avail_v = spu_sub(avail_v, one_at_1);
right++;
// modulus hack
cmp_v2 = spu_cmpeq(*right_tail_idx, size_v);
if( __builtin_expect( spu_extract(cmp_v2,0) ,0) ){
*right_tail_idx = zeros;
right = (vector signed int*) &md[am].buffer[RIGHT][0];
}
} else {
*right = *left;
*left_tail_idx = spu_add(*left_tail_idx,ones);
avail_v = spu_sub(avail_v, one_at_0);
left++;
// modulus hack
cmp_v2 = spu_cmpeq(*left_tail_idx, size_v);
if( __builtin_expect( spu_extract(cmp_v2,0) ,0) ){
*left_tail_idx = zeros;
left = (vector signed int*) &md[am].buffer[LEFT][0];
}
}
// update out head idx
*out_head_idx = spu_add(*out_head_idx,ones);
avail_v = spu_sub(avail_v, one_at_2);
out++;
// modulus hack
cmp_v2 = spu_cmpeq(*out_head_idx, size_v);
if( __builtin_expect(spu_extract(cmp_v2,0),0) ){
out = (vector signed int*) &md[am].buffer[OUT][0];
*out_head_idx = zeros;
}
// is there data still available?
avail = spu_gather(spu_cmpgt(avail_v, zeros));
}
#ifdef TRACE_TIME
merge_loop_ticks += -(spu_read_decrementer() - dec_val2);
#endif
// how much got produced?
vector signed int consumed = spu_sub(avail_before, avail_v);
int consumed_left = spu_extract(consumed, 0);
int consumed_right = spu_extract(consumed, 1);
if(consumed_left)
update_tail(LEFT);
if(consumed_right)
update_tail(RIGHT);
md[am].consumed[LEFT] += consumed_left;
md[am].consumed[RIGHT] += consumed_right;
if(md[am].consumed[LEFT] == mcb[am].data_size[LEFT])
md[am].depleted[LEFT] = 1;
if(md[am].consumed[RIGHT] == mcb[am].data_size[RIGHT])
md[am].depleted[RIGHT] = 1;
if(mcb[am].local[OUT] < 255 && md[am].depleted[LEFT] && md[am].depleted[RIGHT]){
md[am].done = 1;
--num_active_mergers;
}
}
