static void Send_status_query()
{
int32 proc_id;
proc p;
int proc_index;
int i,j;
Pack.type = STATUS_TYPE;
Pack.type = Set_endian( Pack.type );
Pack.conf_hash = MONITOR_HASH;
Pack.data_len= 0;
Pack_scat.num_elements = 1;
Alarm( PRINT , "Monitor: send status query\n");
for( i=0; i < Cn.num_segments ; i++ )
{
for( j=0; j < Cn.segments[i].num_procs; j++ )
{
proc_id = Cn.segments[i].procs[j]->id;
proc_index = Conf_proc_by_id( proc_id, &p );
if( Status_vector[proc_index] )
{
DL_send( SendChan, p.id, p.port, &Pack_scat );
}
}
}
#ifndef _REENTRANT
E_queue( Send_status_query, 0, NULL, Send_status_timeout );
#endif
}
static void Send_partition()
{
int32 proc_id;
proc p;
int proc_index;
int i,j;
Pack.type = PARTITION_TYPE;
Pack.type = Set_endian( Pack.type );
Pack.conf_hash = MONITOR_HASH;
Pack.data_len= sizeof( Partition );;
Pack_scat.num_elements = 2;
Pack_scat.elements[1].len = sizeof( Partition );
Pack_scat.elements[1].buf = (char *)&Partition;
Alarm( PRINT , "Monitor: send partition\n");
for( i=0; i < Cn.num_segments ; i++ )
{
for( j=0; j < Cn.segments[i].num_procs; j++ )
{
proc_id = Cn.segments[i].procs[j]->id;
proc_index = Conf_proc_by_id( proc_id, &p );
DL_send( SendChan, p.id, p.port, &Pack_scat );
DL_send( SendChan, p.id, p.port, &Pack_scat );
}
}
#ifndef _REENTRANT
E_queue( Send_partition, 0, NULL, Send_partition_timeout );
#endif
}
/* 5 = Terminate */
int main(int argc, char **argv)
{
/* Variables */
unsigned char ttl_val;
int c, ret, num_groups;
int complete = 0;
int response[10];
char sender[MAX_GROUP_NAME];
int service_type = 0;
int rts = 0;
char mess_buf[MAX_MESS_LEN];
struct timeval timeout, start_time, end_time;
struct initializers *i=malloc(sizeof(struct initializers));
struct packet_structure *p=malloc(sizeof(struct packet_structure));
sp_time test_timeout;
sp_time delta_time;
int16 mess_type;
int endian_mismatch=0;
char target_groups[MAX_MEMBERS][MAX_GROUP_NAME];
i->debug = 0; /*Turn on for testing */
strcpy(group, "shamil22");
parseargs(argc, argv, i);
setup(i); /*Setup ports and wait for start process */
printf("Waiting to start.\n");
while(!rts) {
ret = SP_receive( Mbox, &service_type, sender, 100, &num_groups, target_groups,
&mess_type, &endian_mismatch, sizeof(mess_buf), mess_buf );
if (ret > 0) {
i->packet = (struct packet_structure *)mess_buf;
if (i->packet->type == 2) {
rts =1;
}
}
}
printf("Begin!\n");
gettimeofday(&start_time, NULL);
starttime1=start_time.tv_sec+(start_time.tv_usec/1000000.0);
delta_time.sec = 0; delta_time.usec =0;
E_queue( send_data, 0, NULL, delta_time );
E_attach_fd( Mbox, READ_FD, receive_packet, 0, NULL, HIGH_PRIORITY );
printf("Handling events.\n");
//send_data(); /*Send first chunk of message*/
E_handle_events();
return 0;
}
void Spread_reconnect(int ret){
if(ret != -8 && ret != -11 && ret != -2)
wack_alarm(EXIT, "Spread_reconnnect: Unexpected Error (%d)", ret);
wack_alarm(PRINT,"connecting to %s", Spread_name);
Clean_up();
if(Mbox >= 0) {
SP_disconnect(Mbox);
E_detach_fd( Mbox, READ_FD );
Mbox = -1;
}
/* connecting to the relevant Spread daemon, asking for group info */
if(spread_lock) {
handle_reconnect(0, NULL);
return;
}
ret = SP_connect( Spread_name, User, 0, 1, &Mbox, Private_group ) ;
if(ret == ACCEPT_SESSION) {
ret = SP_join( Mbox, Spread_group );
if( ret < 0 ) {
SP_error( ret );
SP_disconnect( Mbox );
Mbox = -1;
wack_alarm(PRINT, "Spread join on reconnect failed [%d].", ret);
handle_reconnect(0, NULL);
return;
}
} else {
wack_alarm(PRINT, "Spread connect failed [%d].", ret);
handle_reconnect(0, NULL);
return;
}
/* State initializations */
State = BOOT;
Old_maturity = 0;
Maturity = 0;
E_queue( Turn_mature, 0, 0, Maturity_timeout );
My.num_allocated = 0;
strcpy(My.private_group_name, Private_group);
E_attach_fd( Mbox, READ_FD, Handle_network, 0, NULL, HIGH_PRIORITY );
E_set_active_threshold( HIGH_PRIORITY );
}
static void Handle_mature()
{
if (Maturity == 1) return;
Maturity = 1;
E_dequeue( Turn_mature, 0, 0 );
wack_alarm(LOGIC,"Handle_mature");
if ( State != GATHER )
{
/* ### deterministically grab ip addresses */
Priority_claim();
Claim_unclaimed();
wack_alarm(LOGIC, "Shifting to RUN in Handle_mature()" );
State = RUN;
E_queue( Balance, 0, 0, Balance_timer );
}
}
/* Basic algorithm:
* 1) have configuration code load new conf file and check for modifications to conf.
* 2) If only add/sub of daemons, then initiate membership change with token_loss and return;
* 3) else, then set Conf_reload_state, create singleton partition, and schedule token_loss.
* 4) When membership completes in Discard_packets() cleanup partition and probe for new members.
*/
void Prot_initiate_conf_reload( int code, void *data )
{
bool need_memb_partition;
int16 singleton_partition[MAX_PROCS_RING];
int i;
if (Memb_state() != OP ) {
/* This is a race condition, that the Prot_initiate_conf_reload was scheduled when OP state,
* but another membership occured before it was executed.
* The membership system will requeue this function when it reaches OP state again.
*/
return;
}
/* Disable queueing of this function when OP state reached in membership */
Prot_clear_need_conf_reload();
need_memb_partition = Conf_reload_initiate();
/* Signal all subsystems to update Conf and My strucures */
Net_signal_conf_reload();
Memb_signal_conf_reload();
Sess_signal_conf_reload();
/* update protocol varialbes with new conf */
My = Conf_my();
My_index = Conf_proc_by_id( My.id, &My );
if (need_memb_partition) {
/* make partition */
for ( i = 0 ; i < Conf_num_procs( Conf_ref() ); i++ )
{
singleton_partition[i] = i;
}
Net_set_partition(singleton_partition);
Conf_reload_singleton_state_begin();
}
E_queue( Memb_token_loss, 0, NULL, Zero_timeout );
}
void Prot_token_hurry()
{
/* asked to send token again (almost lost) */
sys_scatter retrans_token;
int32 val;
retrans_token.num_elements = 1;
retrans_token.elements[0].len = sizeof(token_header);
retrans_token.elements[0].buf = (char *)Last_token;
Last_token->rtr_len=0;
if( Conf_leader( Memb_active_ptr() ) == My.id )
{
val = Get_retrans(Last_token->type);
val++;
Last_token->type = Set_retrans( Last_token->type, val );
E_queue( Prot_token_hurry, 0, NULL, Hurry_timeout );
GlobalStatus.token_hurry++;
}
/* sending token */
Net_send_token( &retrans_token );
if( Wide_network &&
Conf_seg_last(Memb_active_ptr(), My.seg_index) == My.id )
{
/* sending again to another segment */
Net_send_token( &retrans_token );
}
if( Get_retrans( Last_token->type ) > 1 )
{
/* problems */
Net_send_token( &retrans_token );
Net_send_token( &retrans_token );
}
Alarm( PROTOCOL, "Prot_token_hurry: retransmiting token %d %d\n",
Get_arq(Last_token->type), Get_retrans(Last_token->type) );
}
void Prot_handle_token(channel fd, int dummy, void *dummy_p)
{
int new_ptr;
int num_retrans, num_allowed, num_sent;
int flow_control;
char *new_rtr;
ring_rtr *ring_rtr_ptr;
int32 rtr_proc_id;
int16 rtr_seg_index;
int32 val;
int retrans_allowed; /* how many of my retrans are allowed on token */
int i, ret;
/* int r1,r2;*/
ret = Net_recv_token( fd, &New_token );
/* from another monitor component */
if( ret == 0 ) return;
/* delete random
r1 = ((-My.id)%17)+3;
r2 = get_rand() % (r1+3 );
if ( r2 == 0 ) return; */
Alarm( DEBUG, "Received Token\n");
/* check if it is a regular token */
if( Is_form( Token->type ) )
{
Alarm(PROTOCOL, "it is a Form Token.\n");
Memb_handle_token( &New_token );
return;
}
/* The Veto property for tokens - swallow this token */
if( ! Memb_token_alive() ) {
Alarm(PROTOCOL, "Prot_handle_token: Veto Property. Memb not alive.\n");
return;
}
if( ret != sizeof(token_header) + Token->rtr_len )
{
Alarm( PRINT,
"Prot_handle_token: recv token len is %d, should be %d\n",
ret,sizeof(token_header) + Token->rtr_len );
return;
}
if( !Same_endian( Token->type ) )
Flip_token_body( New_token.elements[1].buf, Token );
/* Deal with wrapping seq values (2^32) by triggering a membership by dropping the token */
if( (Memb_state() != EVS ) && (Token->seq > MAX_WRAP_SEQUENCE_VALUE ) )
{
Alarm( PRINT, "Prot_handle_token: Token Sequence number (%ld) approaching 2^31 so trigger membership to reset it.\n", Token->seq);
return;
}
if( Conf_leader( Memb_active_ptr() ) == My.id )
{
if( Get_arq(Token->type) != Get_arq(Last_token->type) )
{
Alarm( PROTOCOL,
"Prot_handle_token: leader swallowing token %d %d %d\n",
Get_arq(Token->type),Get_retrans(Token->type),Get_arq(Last_token->type) );
/* received double token - swallow it */
return;
}
}else{
if( Get_arq(Token->type) == Get_arq(Last_token->type) )
{
if( Get_retrans(Token->type) > Get_retrans(Last_token->type) )
{
val = Get_retrans(Token->type);
Last_token->type = Set_retrans(Last_token->type,val);
/* asked to send token again (almost lost) */
Alarm( PROTOCOL,
"Prot_handle_token: not leader, asked to retrans %d %d\n",
Get_arq(Token->type), val );
Prot_token_hurry();
}else{
Alarm( PROTOCOL,
"Prot_handle_token: not leader, swallow same token %d %d\n",
Get_arq(Token->type), Get_retrans(Token->type) );
}
return;
} else if ( Get_arq(Token->type) != ( ( Get_arq( Last_token->type ) + 1 ) % 0x10 ) ) {
Alarm( PROTOCOL,
"Prot_handle_token: not leader, swallowing very outdated token: ARQ(%d) RETRANS(%d) vs. Last ARQ(%d)\n",
Get_arq(Token->type), Get_retrans(Token->type), Get_arq(Last_token->type) );
return;
} else {
if ( Get_retrans(Token->type) > 0 ) {
GlobalStatus.token_hurry++;
}
}
}
if( Highest_seq < Token->seq ) Highest_seq = Token->seq;
/* Handle retransmissions */
num_retrans = Answer_retrans( &new_ptr, &rtr_proc_id, &rtr_seg_index );
GlobalStatus.retrans += num_retrans;
new_rtr = New_token.elements[1].buf;
/* Handle new packets */
flow_control = (int) Token->flow_control;
num_allowed = FC_allowed( flow_control, num_retrans );
num_sent = Send_new_packets( num_allowed );
GlobalStatus.packet_sent += num_sent;
/* Flow control calculations */
Token->flow_control = Token->flow_control
- Last_num_retrans - Last_num_sent
+ num_retrans + num_sent;
Last_num_retrans = num_retrans;
Last_num_sent = num_sent;
/* Prepare my retransmission requests */
for( i = My_aru+1; i <= Highest_seq; i++ )
{
if( ! Packets[i & PACKET_MASK].exist ) break;
My_aru++;
}
GlobalStatus.my_aru = My_aru;
if( My_aru < Highest_seq )
{
/* Compute how many of my retransmission requests are possible to fit */
retrans_allowed = ( sizeof( token_body ) - new_ptr - sizeof( ring_rtr ) ) / sizeof( int32 );
if( retrans_allowed > 1 )
{
ring_rtr_ptr = (ring_rtr *)&new_rtr[new_ptr];
ring_rtr_ptr->memb_id = Memb_id();
ring_rtr_ptr->proc_id = rtr_proc_id;
ring_rtr_ptr->seg_index = rtr_seg_index;
ring_rtr_ptr->num_seq = 0;
new_ptr += sizeof(ring_rtr);
for( i=My_aru+1; i <= Highest_seq && retrans_allowed > 0; i++, retrans_allowed-- )
{
if( ! Packets[i & PACKET_MASK].exist )
{
memcpy( &new_rtr[new_ptr], &i, sizeof(int32) );
new_ptr += sizeof(int32);
ring_rtr_ptr->num_seq++;
}
}
}
}
if( Memb_state() == EVS )
{
if( My_aru == Highest_seq )
{
My_aru = Last_seq;
Memb_commit();
}
}
Token->rtr_len = new_ptr;
New_token.elements[1].len = new_ptr;
/* Calculating Token->aru and Set_aru */
if( ( Token->aru == Set_aru ) ||
( Token->aru_last_id == My.id ) ||
( Token->aru == Token->seq ) )
{
Token->aru = My_aru;
Token->aru_last_id = My.id;
if( My_aru < Highest_seq ) Set_aru = My_aru;
else Set_aru = -1;
}else if( Token->aru > My_aru ) {
Token->aru = My_aru;
Token->aru_last_id = My.id;
Set_aru = My_aru;
}else{
Set_aru = -1;
}
Token->proc_id = My.id;
if( Memb_state() != EVS ) Token->seq = Highest_seq;
if( Conf_leader( Memb_active_ptr() ) == My.id )
{
val = Get_arq( Token->type );
val = (val + 1)% 0x10;
Token->type = Set_arq( Token->type, val );
Token->type = Set_retrans( Token->type, 0 );
}
/* Send token */
if( ! ( Conf_leader( Memb_active_ptr() ) == My.id &&
To_hold_token() ) )
{
/* sending token */
Net_send_token( &New_token );
/* ### Bug fix for SGIs */
#ifdef ARCH_SGI_IRIX
Net_send_token( &New_token );
#endif /* ARCH_SGI_IRIX */
if( Get_retrans( Token->type ) > 1 )
{
/* problems */
Net_send_token( &New_token );
Net_send_token( &New_token );
}
}
Token_rounds++;
if( Conf_leader( Memb_active_ptr() ) == My.id )
E_queue( Prot_token_hurry, 0, NULL, Hurry_timeout );
E_queue( Memb_token_loss, 0, NULL, Token_timeout );
/* calculating Aru */
if( Token->aru > Last_token->aru )
Aru = Last_token->aru;
else
Aru = Token->aru;
if( Highest_seq == Aru ) Token_counter++;
else Token_counter = 0;
dispose( Last_token );
Last_token = Token;
New_token.elements[0].buf = (char *) new(TOKEN_HEAD_OBJ);
New_token.elements[1].len = sizeof(token_body);
Token = (token_header *)New_token.elements[0].buf;
/* Deliver & discard packets */
Discard_packets();
Deliver_agreed_packets();
Deliver_reliable_packets( Highest_seq-num_sent+1, num_sent );
if( Memb_state() == EVS && Token_rounds > MAX_EVS_ROUNDS )
{
Alarmp( SPLOG_ERROR, PRINT, "Prot_handle_token: BUG WORKAROUND: Too many rounds in EVS state; swallowing token; state:\n" );
Alarmp( SPLOG_ERROR, PRINT, "\tAru: %d\n", Aru );
Alarmp( SPLOG_ERROR, PRINT, "\tMy_aru: %d\n", My_aru );
Alarmp( SPLOG_ERROR, PRINT, "\tHighest_seq: %d\n", Highest_seq );
Alarmp( SPLOG_ERROR, PRINT, "\tHighest_fifo_seq: %d\n", Highest_fifo_seq );
Alarmp( SPLOG_ERROR, PRINT, "\tLast_discarded: %d\n", Last_discarded );
Alarmp( SPLOG_ERROR, PRINT, "\tLast_delivered: %d\n", Last_delivered );
Alarmp( SPLOG_ERROR, PRINT, "\tLast_seq: %d\n", Last_seq );
Alarmp( SPLOG_ERROR, PRINT, "\tToken_rounds: %d\n", Token_rounds );
Alarmp( SPLOG_ERROR, PRINT, "Last Token:\n" );
Alarmp( SPLOG_ERROR, PRINT, "\ttype: 0x%x\n", Last_token->type );
Alarmp( SPLOG_ERROR, PRINT, "\ttransmiter_id: %d\n", Last_token->transmiter_id );
Alarmp( SPLOG_ERROR, PRINT, "\tseq: %d\n", Last_token->seq );
Alarmp( SPLOG_ERROR, PRINT, "\tproc_id: %d\n", Last_token->proc_id );
Alarmp( SPLOG_ERROR, PRINT, "\taru: %d\n", Last_token->aru );
Alarmp( SPLOG_ERROR, PRINT, "\taru_last_id: %d\n", Last_token->aru_last_id );
Alarmp( SPLOG_ERROR, PRINT, "\tflow_control: %d\n", Last_token->flow_control );
Alarmp( SPLOG_ERROR, PRINT, "\trtr_len: %d\n", Last_token->rtr_len );
Alarmp( SPLOG_ERROR, PRINT, "\tconf_hash: %d\n", Last_token->conf_hash );
Memb_token_loss();
}
GlobalStatus.highest_seq = Highest_seq;
GlobalStatus.aru = Aru;
GlobalStatus.token_rounds = Token_rounds;
}
static void Prot_handle_bcast(channel fd, int dummy, void *dummy_p)
{
packet_header *pack_ptr;
int pack_entry;
proc p;
int received_bytes;
int total_bytes_processed;
int num_buffered_packets;
int i;
int32 j;
/* int r1,r2; */
received_bytes = Net_recv( fd, &New_pack );
/* My own packet or from another monitor component */
if( received_bytes == 0 ) return;
/* problem in receiving */
if( received_bytes < 0 ) return;
pack_ptr = (packet_header *)New_pack.elements[0].buf;
/* ### Pack, this has to move down to network.c
* if( pack_ptr->data_len +sizeof(packet_header) != received_bytes )
* {
* Alarm( PRINT, "Prot_handle_bcast: received %d, should be %d\n",
* received_bytes, pack_ptr->data_len+sizeof(packet_header) );
* return;
* }
*/
if( Is_status( pack_ptr->type ) )
{
Stat_handle_message( &New_pack );
return;
}
if( Is_fc( pack_ptr->type ) )
{
FC_handle_message( &New_pack );
return;
}
if( Is_conf_reload( pack_ptr->type ) )
{
Prot_handle_conf_reload( &New_pack );
return;
}
/* delete random
r1 = ((-My.id)%17)+3;
r2 = get_rand() % (r1+3 );
if ( r2 == 0 ) return; */
if( Is_membership( pack_ptr->type ) )
{
Memb_handle_message( &New_pack );
return;
}
if( Is_hurry( pack_ptr->type ) )
{
Handle_hurry( pack_ptr );
return;
}
if( !Is_regular( pack_ptr->type ) )
{
Alarm( PROTOCOL, "Prot_handle_bcast: Unknown packet type %d\n",
pack_ptr->type );
return;
}
if( ! Memb_is_equal( Memb_id(), pack_ptr->memb_id ) )
{
/* Foreign message */
Memb_handle_message( &New_pack );
return;
}
if (Memb_token_alive() ) {
E_queue( Memb_token_loss, 0, NULL, Token_timeout );
if( Conf_leader( Memb_active_ptr() ) == My.id )
{
E_queue( Prot_token_hurry, 0, NULL, Hurry_timeout );
}
}
/* ### Pack: next 70 lines (almost till the end of the routine) have changed */
Buffered_packets[0].head = pack_ptr;
Buffered_packets[0].body = (packet_body *)New_pack.elements[1].buf;
received_bytes -= sizeof(packet_header);
total_bytes_processed = pack_ptr->data_len;
/* ignore any alignment padding */
switch(total_bytes_processed % 4)
{
case 1:
total_bytes_processed++;
case 2:
total_bytes_processed++;
case 3:
total_bytes_processed++;
case 0:
/* already aligned */
break;
}
for( i = 1; received_bytes > total_bytes_processed; i++ )
{
/* copy into each of the elements after the first element*/
Buffered_packets[i].head = (packet_header *)new(PACK_HEAD_OBJ);
Buffered_packets[i].body = (packet_body *)new(PACKET_BODY);
if (Buffered_packets[i].head == NULL)
Alarm(EXIT, "Prot_handle_bcast: Memory allocation failed for PACK_HEAD_OBJ\n");
if (Buffered_packets[i].body == NULL)
Alarm(EXIT, "Prot_handle_bcast: Memory allocation failed for PACKET_BODY\n");
pack_ptr = (packet_header *)&New_pack.elements[1].buf[total_bytes_processed];
memcpy( Buffered_packets[i].head, pack_ptr, sizeof( packet_header ) );
total_bytes_processed += sizeof(packet_header);
memcpy( Buffered_packets[i].body, &New_pack.elements[1].buf[total_bytes_processed], pack_ptr->data_len);
total_bytes_processed += pack_ptr->data_len;
/* ignore any alignment padding */
switch(total_bytes_processed % 4)
{
case 1:
total_bytes_processed++;
case 2:
total_bytes_processed++;
case 3:
total_bytes_processed++;
case 0:
/* already aligned */
break;
}
}
num_buffered_packets = i;
for( i = 0; i < num_buffered_packets; i++)
{
pack_ptr = Buffered_packets[i].head;
/* do we have this packet */
if( pack_ptr->seq <= Aru )
{
Alarm( PROTOCOL, "Prot_handle_bcast: delayed packet %d already delivered (Aru %d)\n",
pack_ptr->seq, Aru );
dispose(Buffered_packets[i].head);
dispose(Buffered_packets[i].body);
continue;
}
pack_entry = pack_ptr->seq & PACKET_MASK;
if( Packets[pack_entry].exist )
{
Alarm( PROTOCOL, "Prot_handle_bcast: packet %d already exist\n",
pack_ptr->seq );
dispose(Buffered_packets[i].head);
dispose(Buffered_packets[i].body);
continue;
}
Packets[pack_entry].proc_index = Conf_proc_by_id( pack_ptr->proc_id, &p );
if( Packets[pack_entry].proc_index < 0 )
{
Alarm( PROTOCOL, "Prot_handle_bcast: unknown proc %d\n", pack_ptr->proc_id );
dispose(Buffered_packets[i].head);
dispose(Buffered_packets[i].body);
continue;
}
/* insert new packet */
Packets[pack_entry].head = pack_ptr;
Packets[pack_entry].body = Buffered_packets[i].body;
Packets[pack_entry].exist = 1;
/* update variables */
if( Highest_seq < pack_ptr->seq ) Highest_seq = pack_ptr->seq;
if( pack_ptr->seq == My_aru+1 )
{
for( j=pack_ptr->seq; j <= Highest_seq; j++ )
{
if( ! Packets[j & PACKET_MASK].exist ) break;
My_aru++;
}
Deliver_agreed_packets();
}else Deliver_reliable_packets( pack_ptr->seq, 1 );
Alarm( PROTOCOL, "Prot_handle_bcast: packet %d inserted\n",
pack_ptr->seq );
} /* END OF LOOP */
GlobalStatus.packet_recv++;
GlobalStatus.my_aru = My_aru;
GlobalStatus.highest_seq = Highest_seq;
/* prepare New_pack for next packet */
New_pack.elements[0].buf = (char *) new(PACK_HEAD_OBJ);
New_pack.elements[1].buf = (char *) new(PACKET_BODY);
}
void Discard_packets()
{
int pack_entry;
packet_body *body_ptr;
up_queue *up_ptr;
int proc_index;
int i;
if( Aru <= Last_discarded ) return;
if( Memb_state() == EVS )
{
int found_hole;
membership_id reg_memb_id;
if( Aru != Last_seq ) return;
/* Deliver packets that must be delivered before the transitional signal.
* Those up to the Aru for my old ring were delivered in Read_form2().
* So, it remains to deliver all packets up to the first hole or the first
* SAFE message. */
Alarmp( SPLOG_INFO, PROTOCOL,
"Discard_packets: delivering messages after old ring Aru before transitional\n" );
for( i = Last_discarded+1; i <= Highest_seq; i++ )
{
pack_entry = i & PACKET_MASK;
if( ! Packets[pack_entry].exist )
Alarmp( SPLOG_FATAL, PROTOCOL, "Discard_packets: (EVS before transitional) packet %d not exist\n", i);
if( Packets[pack_entry].exist == 3 )
{
Alarmp( SPLOG_INFO, PROTOCOL, "Discard_packets: Found first Hole in %d\n", i);
break;
}
if( Is_safe( Packets[pack_entry].head->type ) ) {
Alarmp( SPLOG_INFO, PROTOCOL, "Discard_packets: Found first SAFE message in %d", i);
break;
}
/* should deliver packet or dispose the body if it was delivered already */
if( Packets[pack_entry].exist == 1 ){
Deliver_packet( pack_entry, 0 );
} else {
dispose( Packets[pack_entry].body );
}
/* dispose packet header in any case */
dispose( Packets[pack_entry].head );
Alarmp( SPLOG_INFO, PROTOCOL, "Discard_packets: delivering %d in EVS\n",i);
Packets[pack_entry].exist = 0;
Last_discarded = i;
}
/* calculate and deliver transitional membership */
Alarmp( SPLOG_INFO, PROTOCOL, "Discard_packets: Delivering transitional membership\n" );
Memb_transitional();
Sess_deliver_trans_memb( Trans_membership, Memb_trans_id() );
/* deliver all remaining packets for EVS */
found_hole = 0;
for( i = Last_discarded+1; i <= Highest_seq; i++ )
{
pack_entry = i & PACKET_MASK;
if( ! Packets[pack_entry].exist )
Alarm( EXIT, "Discard_packets: (EVS after transitional) packet %d not exist\n", i);
if( Packets[pack_entry].exist == 3 )
{
/*
* There is a hole!
* from here, we need to check if the proc_id of the packet
* is in commited membership.
*/
found_hole = 1;
Alarm( PROTOCOL, "Discard_packets: Found a Hole in %d \n",i);
}else if( (!found_hole) ||
(Conf_id_in_conf( &Commit_membership, Packets[pack_entry].head->proc_id ) != -1) ){
/* should deliver packet or dispose the body if it was delivered already */
if( Packets[pack_entry].exist == 1 ){
Deliver_packet( pack_entry, 0 );
}else{
dispose( Packets[pack_entry].body );
}
/* dispose packet header in any case */
dispose( Packets[pack_entry].head );
Alarm( PROTOCOL, "Discard_packets: delivering %d in EVS\n",i);
}else{
/* should not deliver packet */
dispose( Packets[pack_entry].head );
dispose( Packets[pack_entry].body );
Alarm( PROTOCOL, "Discard_packets: Due to hole, not delivering %d \n",i);
}
Packets[pack_entry].exist = 0;
}
/* check up_queue and down_queue */
if( Down_queue_ptr->num_mess > 0 )
{
Down_queue_ptr->cur_element = 0;
}
for( proc_index=0; proc_index < MAX_PROCS_RING; proc_index++ )
{
if( Up_queue[proc_index].exist )
{
if( proc_index != My_index )
{
/*
* dispose only packets that are not mine
* my packets will stay in Down_queue if the message is not
* ready to be delivered (because not fully sent yet)
* so we need not to dispose them!
*/
up_ptr = &Up_queue[proc_index];
for( i=0; i < up_ptr->mess->num_elements; i++ )
{
body_ptr = (packet_body *)up_ptr->mess->elements[i].buf;
dispose( body_ptr );
}
}
dispose( Up_queue[proc_index].mess );
Up_queue[proc_index].exist = 0;
}
}
/* calculate and deliver regular membership */
Memb_regular();
Log_membership();
reg_memb_id = Memb_id();
Sess_deliver_reg_memb( Reg_membership, reg_memb_id );
/* If in change conf mode; then if singleton (which should be true) and GOP state then:
* Remove partition
* Initiate Memb_lookup() to find other daemons
*/
if( Conf_in_reload_singleton_state() ) {
/* GOP state equals value 1, but is private declaration in groups.c */
if ( (GlobalStatus.gstate != 1 ) || ( Conf_num_procs( &Reg_membership ) != 1 ) ) {
Alarmp( SPLOG_FATAL, MEMB, "Discard_packets: Failed to reload configuration - gstate: %d and num_procs in membership: %d\n", GlobalStatus.gstate, Conf_num_procs( &Reg_membership) );
}
Net_clear_partition();
E_queue( Memb_lookup_new_members, 0, NULL, Zero_timeout);
Conf_reload_singleton_state_end();
}
/* set variables for next membership */
if ( Conf_debug_initial_sequence() ) {
Last_token->aru = INITIAL_SEQUENCE_NEAR_WRAP;
Highest_seq = INITIAL_SEQUENCE_NEAR_WRAP;
Highest_fifo_seq = INITIAL_SEQUENCE_NEAR_WRAP;
My_aru = INITIAL_SEQUENCE_NEAR_WRAP;
Aru = INITIAL_SEQUENCE_NEAR_WRAP;
Set_aru = INITIAL_SEQUENCE_NEAR_WRAP -1;
Last_discarded = INITIAL_SEQUENCE_NEAR_WRAP;
Last_delivered = INITIAL_SEQUENCE_NEAR_WRAP;
} else {
Last_token->aru = 0;
Highest_seq = 0;
Highest_fifo_seq = 0;
My_aru = 0;
Aru = 0;
Set_aru = -1;
Last_discarded = 0;
Last_delivered = 0;
}
GlobalStatus.my_aru = My_aru;
Token_counter = 0;
}else{
for( i = Last_discarded+1; i <= Aru; i++ )
{
pack_entry = i & PACKET_MASK;
if( ! Packets[pack_entry].exist )
Alarm( EXIT, "Discard_packets: (NOT EVS) packet %d not exist\n",i);
/* should deliver packet or dispose the body if it was delivered already */
if( Packets[pack_entry].exist == 1 ) Deliver_packet( pack_entry, 0 );
else dispose( Packets[pack_entry].body );
/* dispose packet header in any case */
dispose( Packets[pack_entry].head );
Packets[pack_entry].exist = 0;
}
Alarm( PROTOCOL, "Discard_packets: packets %d-%d were discarded\n",
Last_discarded+1, Aru );
Last_discarded = Aru;
if( Last_delivered < Last_discarded ) Last_delivered = Last_discarded;
}
}
void receive_packet() {
/* receiving data */
int c, ret, num_groups;
int r=1;
int16 mess_type;
int endian_mismatch=0;
char mess_buf[MAX_MESS_LEN];
char target_groups[MAX_MEMBERS][MAX_GROUP_NAME];
char sender[MAX_GROUP_NAME];
int service_type = 0;
struct timeval end_time;
struct packet_structure *packet;
SP_receive( Mbox, &service_type, sender, 100, &num_groups, target_groups,
&mess_type, &endian_mismatch, sizeof(mess_buf), mess_buf );
packet = (struct packet_structure *)mess_buf;
if (packet->type == 1) /*Data packet, write to log */
{
write_log(packet);
}
else if (packet->type == 3 && machine_index == 1)
{ printf("Checking for termination \n");
completed[packet->machine_index] = 1;
for (c=1; c <= total_machines; c++) {
if (completed[c] == 0) r =0;
printf("%d = %d\n", c, completed[c]);
}
if (r==1) {
/*All machines complete. Send termination */
packet->type = 5; printf("Complete sending termination\n");
ret= SP_multicast( Mbox, AGREED_MESS, group, 1, sizeof(struct packet_structure), (char *)packet );
}
}
else if (packet->type == 5) /*Terminate */
{
printf("Complete\n");
gettimeofday(&end_time, NULL);
starttime2=end_time.tv_sec+(end_time.tv_usec/1000000.0);
printf("%.6lf seconds elapsed\n", starttime2-starttime1);
fclose(logfile);
Bye();
exit (0);
}
else
{
printf("Got packet type %d, mid=%d\n", packet->type, packet->machine_index);
printf("received message of unknown message type 0x%x with ret %d\n", service_type, ret);
}
sendcount++; /* Only send when we've received at least what we've sent. */
if (packets_to_send > 0 && sendcount == FCC) {
sendcount = 0;
sp_time delta_time;
delta_time.sec = 0; delta_time.usec =1250; /*Setting this below 1000 causes problems */
E_queue( send_data, 0, NULL, delta_time ); /*Queue up the sender */
}
if (packets_to_send == 0 && completed[machine_index] == 0){ /*Send we are complete if we haven't already */
printf("Sending completed\n");
completed[machine_index] = 1;
packet->type = 3;
packet->machine_index = machine_index;
ret= SP_multicast( Mbox, AGREED_MESS, group, 1, sizeof(struct packet_structure), (char *)packet );
}
}
static void Balance(int __code, void *__data)
{
int ret;
int num_bytes;
int i, j, k;
int min_index;
int max_index;
static member *tempMembers = NULL;
static entry *tempTable = NULL;
wack_alarm(LOGIC, "Balance called." );
if(!tempMembers) {
tempMembers = (member *)malloc(sizeof(member) * MAX_PSEUDO);
}
if(!tempTable) {
tempTable = (entry *)malloc(sizeof(entry) * MAX_PSEUDO);
}
if( My_index ) goto leave;
if( State != RUN ) return;
State = BALANCE;
num_bytes = sizeof( int );
min_index = 0;
max_index = 0;
memcpy( tempMembers, Members, MAX_PSEUDO * sizeof( member ) );
memcpy( tempTable, Allocation_table, MAX_PSEUDO * sizeof( entry ) );
for ( k = 0; k < Balance_rate; k++ )
{
for ( i = 0; i < Num_members; i++ )
{
if( tempMembers[i].num_allocated < tempMembers[min_index].num_allocated )
min_index = i;
if( tempMembers[i].num_allocated > tempMembers[max_index].num_allocated )
max_index = i;
}
if( tempMembers[max_index].num_allocated - tempMembers[min_index].num_allocated > 1 )
{
for( j = 0; j < Num_pseudo; j++ )
{
if( _rif_ip_s(tempTable[j]) == _rif_ip_s(tempMembers[max_index])
&& tempTable[j].claim_priority < PREFER )
{
memcpy( &Mess[num_bytes], &(_rif_ip_s(tempMembers[min_index])), sizeof( address ) );
num_bytes += sizeof( address );
memcpy( &Mess[num_bytes], &(_pif_ip_s(tempTable[j])), sizeof( address ) );
num_bytes += sizeof( address );
tempMembers[min_index].num_allocated++;
tempMembers[max_index].num_allocated--;
_rif_ip_s(tempTable[j]) = _rif_ip_s(tempMembers[min_index]);
j = Num_pseudo;
}
else if ( j == Num_pseudo )
{
if( k == 0 )
{
State = RUN;
goto leave;
}
memcpy( &Mess[0], &k, sizeof( int ) );
k = Balance_rate + 1;
}
}
}
else
{
if( k == 0 )
{
State = RUN;
goto leave;
}
memcpy( &Mess[0], &k, sizeof( int ) );
k = Balance_rate + 1;
}
}
if ( k == Balance_rate )
memcpy( &Mess[0], &k, sizeof( int ) );
ret = SP_multicast( Mbox, AGREED_MESS, Spread_group, BALANCE_MESS, num_bytes, Mess );
if( ret < 0 )
{
SP_error( ret );
Spread_reconnect(ret);
}
State = RUN;
wack_alarm(LOGIC, "Shifting to RUN in balance" );
leave:
if( !Complete_balance )
E_queue( Balance, 0, 0, Balance_timer );
return;
}
static void Send_local_arp_cache_repeat()
{
Send_local_arp_cache(0, NULL);
E_queue( Send_local_arp_cache, 0, 0, ArpRefresh_timer );
}
static void Handle_state()
{
group_id *curr_gid_ptr;
address *curr_real_address_ptr;
int num_bytes;
int num_released;
int *curr_index_ptr;
int *curr_maturity_ptr;
int *curr_num_allocated_ptr;
address *curr_pseudo_addr_ptr;
int *curr_claim_priority_ptr;
int *curr_num_prefer_ptr;
address *curr_prefer_address;
int i, j, k;
wack_alarm(LOGIC, "Handle state" );
num_bytes = 0;
curr_gid_ptr = (group_id *)&Mess[num_bytes];
num_bytes += sizeof( group_id );
/* ### wackamole stuff */
curr_real_address_ptr = (address *)&Mess[num_bytes];
num_bytes += sizeof( address );
curr_index_ptr = (int *)&Mess[num_bytes];
num_bytes += sizeof( int );
curr_maturity_ptr = (int *)&Mess[num_bytes];
num_bytes += sizeof( int );
curr_num_allocated_ptr = (int *)&Mess[num_bytes];
num_bytes += sizeof( int );
if( Endian_mismatch )
{
curr_gid_ptr->id[0] = Flip_int32( curr_gid_ptr->id[0] );
curr_gid_ptr->id[1] = Flip_int32( curr_gid_ptr->id[1] );
curr_gid_ptr->id[2] = Flip_int32( curr_gid_ptr->id[2] );
/* ### wackamole stuff */
*curr_real_address_ptr = Flip_int32( *curr_real_address_ptr );
*curr_index_ptr = Flip_int32( *curr_index_ptr );
*curr_maturity_ptr = Flip_int32( *curr_maturity_ptr );
*curr_num_allocated_ptr = Flip_int32( *curr_num_allocated_ptr );
}
/* Check this here, don't process extra stuff if we don't need to. */
if( Gid.id[0] != curr_gid_ptr->id[0] ||
Gid.id[1] != curr_gid_ptr->id[1] ||
Gid.id[2] != curr_gid_ptr->id[2] ) return;
_rif_ip_s(Members[*curr_index_ptr]) = *curr_real_address_ptr;
Members[*curr_index_ptr].num_allocated = *curr_num_allocated_ptr;
Members[*curr_index_ptr].got_state_from = 1;
num_released = 0;
/* Rip addresses and compute stuff */
for ( i = 0; i < *curr_num_allocated_ptr; i++ )
{
curr_pseudo_addr_ptr = (address *)&Mess[num_bytes];
num_bytes += sizeof( address );
curr_claim_priority_ptr = (int *)&Mess[num_bytes];
num_bytes += sizeof( int );
if( Endian_mismatch )
{
*curr_pseudo_addr_ptr = Flip_int32( *curr_pseudo_addr_ptr );
*curr_claim_priority_ptr = Flip_int32( *curr_claim_priority_ptr );
}
for ( j = 0; j < Num_pseudo; j++)
{
if ( *curr_pseudo_addr_ptr == _pif_ip_s(Allocation_table[j]) )
{
if( _rif_ip_s(Allocation_table[j]) == 0 )
{
_rif_ip_s(Allocation_table[j]) = *curr_real_address_ptr;
Allocation_table[j].claim_priority = *curr_claim_priority_ptr;
}
else
{/*conflict in ip address case*/
if( Allocation_table[j].claim_priority == PREFER_CLAIMED ||
Allocation_table[j].claim_priority == *curr_claim_priority_ptr )
{
Members[*curr_index_ptr].num_allocated--;
if( _rif_ip_s(My) == *curr_real_address_ptr )
{
num_released++;
ReleaseAddress(*curr_pseudo_addr_ptr);
}
Old_table[j].claim_priority = Allocation_table[j].claim_priority;
_rif_ip_s(Old_table[j]) = _rif_ip_s(Allocation_table[j]);
}
else if( Allocation_table[j].claim_priority == CLAIMED &&
*curr_claim_priority_ptr > PREFER )
{
if( _rif_ip_s(My) == _rif_ip_s(Allocation_table[j]) )
{
num_released++;
ReleaseAddress( *curr_pseudo_addr_ptr );
}
for(k = 0; k < Num_members; ++k)
{
if( _rif_ip_s(Members[k]) == _rif_ip_s(Allocation_table[j]) )
Members[k].num_allocated--;
}
_rif_ip_s(Allocation_table[j]) = *curr_real_address_ptr;
Allocation_table[j].claim_priority = *curr_claim_priority_ptr;
Old_table[j].claim_priority = Allocation_table[j].claim_priority;
_rif_ip_s(Old_table[j]) = _rif_ip_s(Allocation_table[j]);
}
}
j = Num_pseudo;
}
}
}
if(num_released) {
Handle_Post_Release();
}
curr_num_prefer_ptr = (int *)&Mess[num_bytes];
num_bytes += sizeof( int );
if( Endian_mismatch ) *curr_num_prefer_ptr = Flip_int32( *curr_num_prefer_ptr );
for ( i = 0; i < *curr_num_prefer_ptr; i++ )
{
curr_prefer_address = (address *)&Mess[num_bytes];
num_bytes += sizeof( address );
if( Endian_mismatch )
*curr_prefer_address = Flip_int32( *curr_prefer_address );
for ( j = 0; j < Num_pseudo; j++)
if ( *curr_prefer_address == _pif_ip_s(Allocation_table[j]) )
{
if ( Allocation_table[j].claim_priority < PREFER )
{
_rif_ip_s(Allocation_table[j]) = *curr_real_address_ptr;
Allocation_table[j].claim_priority = PREFER;
}
j = Num_pseudo;
}
}
/* ### wackamole compute stuff */
if ( Maturity == 0 && *curr_maturity_ptr == 1 )
Handle_mature();
wack_alarm(LOGIC, "handle state: got state from %d", *curr_real_address_ptr );
Num_pending_states--;
if( Num_pending_states == 0 )
{
wack_alarm(LOGIC, "handle state: Finished getting states, maturity is %d.", Maturity );
if ( Maturity )
{
/* wackamole calculate, drop and aquire */
/* Do we do Priority_claim or Claim_unclaimed first? ... each has advantages. */
Priority_claim();
Claim_unclaimed();
State = RUN;
E_queue( Balance, 0, 0, Balance_timer );
wack_alarm(LOGIC, "handle state: shifting to RUN");
}
else
{
State = BOOT;
wack_alarm(LOGIC, "handle state: shifting back to BOOT");
}
Print_alloc();
}
}
void handle_reconnect(int a, void *d) {
sp_time delay;
delay.sec = Spread_retry_interval;
delay.usec = 0;
E_queue((void(*)(int, void *))Spread_reconnect, -2, NULL, delay);
}
