int main(
    int argc,
    char **argv)
{
    int nprocs;
    int rank;
    int num_nets, *net_ids;
    //printf("\n Config count %d ",(int) config.lpgroups_count);
    g_tw_ts_end = s_to_ns(60*60*24*365); /* one year, in nsecs */
    lp_io_handle handle;

    tw_opt_add(app_opt);
    tw_init(&argc, &argv);

    if(argc < 2)
    {
	    printf("\n Usage: mpirun <args> --sync=2/3 mapping_file_name.conf (optional --nkp) ");
	    MPI_Finalize();
	    return 0;
    }
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
  
    configuration_load(argv[2], MPI_COMM_WORLD, &config);
    svr_add_lp_type();
    
    codes_mapping_setup();

    net_ids = model_net_configure(&num_nets);
    assert(num_nets==1);
    net_id = *net_ids;
    free(net_ids);
    
    num_servers = codes_mapping_get_lp_count("MODELNET_GRP", 0, "server",
            NULL, 1);
    if(net_id == DRAGONFLY)
    {
	  num_routers = codes_mapping_get_lp_count("MODELNET_GRP", 0,
                  "dragonfly_router", NULL, 1); 
	  offset = 1;
    }

    if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_WORLD) < 0)
    {
        return(-1);
    }

    tw_run();
    model_net_report_stats(net_id);

    if(lp_io_flush(handle, MPI_COMM_WORLD) < 0)
    {
        return(-1);
    }

    tw_end();
    return 0;
}
Example #2
0
static void node_collective_fan_out(nodes_state * s,
                                    tw_bf * bf,
                                    nodes_message * msg,
                                    tw_lp * lp)
{
    int i;
    //TODO: be annotation-aware
    int num_lps = codes_mapping_get_lp_count(grp_name, 1, LP_CONFIG_NM,
                  NULL, 1);
    bf->c1 = 0;
    bf->c2 = 0;

    send_remote_event(s, bf, msg, lp);

    if(!s->is_leaf)
    {
        bf->c1 = 1;
        tw_event* e_new;
        nodes_message * msg_new;
        tw_stime xfer_to_nic_time;

        for( i = 0; i < s->num_children; i++ )
        {
            xfer_to_nic_time = g_tw_lookahead + TORUS_FAN_OUT_DELAY + tw_rand_exponential(lp->rng, (double)TORUS_FAN_OUT_DELAY/10);

            if(s->children[i] > 0)
            {
                tw_lpid child_nic_id;

                /* get global LP ID of the child node */
                codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, NULL, 1,
                                        s->children[i]/num_lps,
                                        (s->children[i] % num_lps), &child_nic_id);
                //e_new = codes_event_new(child_nic_id, xfer_to_nic_time, lp);
                //msg_new = tw_event_data(e_new);
                //memcpy(msg_new, msg, sizeof(nodes_message) + msg->remote_event_size_bytes);
                void* m_data;
                e_new = model_net_method_event_new(child_nic_id,
                                                   xfer_to_nic_time,
                                                   lp, TORUS, (void**)&msg_new, &m_data);
                memcpy(msg_new, msg, sizeof(nodes_message));
                if (msg->remote_event_size_bytes) {
                    memcpy(m_data, model_net_method_get_edata(TORUS, msg),
                           msg->remote_event_size_bytes);
                }


                msg_new->type = T_COLLECTIVE_FAN_OUT;
                msg_new->sender_node = s->node_id;
                tw_event_send(e_new);
            }
        }
    }
    //printf("\n Fan out phase completed %ld ", lp->gid);
    if(max_collective < tw_now(lp) - s->collective_init_time )
    {
        bf->c2 = 1;
        max_collective = tw_now(lp) - s->collective_init_time;
    }
}
/* handle initial event */
static void handle_kickoff_event(
    svr_state * ns,
    tw_bf * b,
    svr_msg * m,
    tw_lp * lp)
{
    svr_msg * m_local = malloc(sizeof(svr_msg));
    svr_msg * m_remote = malloc(sizeof(svr_msg));

//    m_local->svr_event_type = REQ;
    m_local->svr_event_type = LOCAL;
    m_local->src = lp->gid;

    memcpy(m_remote, m_local, sizeof(svr_msg));
    m_remote->svr_event_type = (do_pull) ? ACK : REQ;
    //printf("handle_kickoff_event(), lp %llu.\n", (unsigned long long)lp->gid);

    /* record when transfers started on this server */
    ns->start_ts = tw_now(lp);

    num_servers_per_rep = codes_mapping_get_lp_count("MODELNET_GRP", 1,
            "server", NULL, 1);
    num_routers_per_rep = codes_mapping_get_lp_count("MODELNET_GRP", 1,
            "dragonfly_router", NULL, 1);

    lps_per_rep = num_servers_per_rep * 2 + num_routers_per_rep;

    int opt_offset = 0;
    int total_lps = num_servers * 2 + num_routers;

    if(net_id == DRAGONFLY && (lp->gid % lps_per_rep == num_servers_per_rep - 1))
          opt_offset = num_servers_per_rep + num_routers_per_rep; /* optional offset due to dragonfly mapping */
    
    /* each server sends a request to the next highest server */
    int dest_id = (lp->gid + offset + opt_offset)%total_lps;
    if (do_pull){
        model_net_pull_event(net_id, "test", dest_id, PAYLOAD_SZ, 0.0,
                sizeof(svr_msg), (const void*)m_remote, lp);
    }
    else{
        model_net_event(net_id, "test", dest_id, PAYLOAD_SZ, 0.0, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp);
    }
    ns->msg_sent_count++;
}
Example #4
0
void torus_collective_init(nodes_state * s,
                           tw_lp * lp)
{
    // TODO: be annotation-aware somehow
    codes_mapping_get_lp_info(lp->gid, grp_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
    int num_lps = codes_mapping_get_lp_count(grp_name, 1, LP_CONFIG_NM, s->anno, 0);
    int num_reps = codes_mapping_get_group_reps(grp_name);
    s->node_id = (mapping_rep_id * num_lps) + mapping_offset;

    int i;
    /* handle collective operations by forming a tree of all the LPs */
    /* special condition for root of the tree */
    if( s->node_id == 0)
    {
        s->parent_node_id = -1;
        s->is_root = 1;
    }
    else
    {
        s->parent_node_id = (s->node_id - ((s->node_id - 1) % TREE_DEGREE)) / TREE_DEGREE;
        s->is_root = 0;
    }
    s->children = (tw_lpid*)malloc(TREE_DEGREE * sizeof(tw_lpid));

    /* set the isleaf to zero by default */
    s->is_leaf = 1;
    s->num_children = 0;

    /* calculate the children of the current node. If its a leaf, no need to set children,
       only set isleaf and break the loop*/

    for( i = 0; i < TREE_DEGREE; i++ )
    {
        tw_lpid next_child = (TREE_DEGREE * s->node_id) + i + 1;
        if(next_child < (num_lps * num_reps))
        {
            s->num_children++;
            s->is_leaf = 0;
            s->children[i] = next_child;
        }
        else
            s->children[i] = -1;
    }

#if TORUS_COLLECTIVE_DEBUG == 1
    printf("\n LP %ld parent node id ", s->node_id);

    for( i = 0; i < TREE_DEGREE; i++ )
        printf(" child node ID %ld ", s->children[i]);
    printf("\n");

    if(s->is_leaf)
        printf("\n LP %ld is leaf ", s->node_id);
#endif
}
Example #5
0
int main(int argc, char **argv) {
  int nprocs;
  int rank;
  int num_nets, *net_ids;
  int i;

  //for later random pairing:
  srand(time(NULL));
  
  g_tw_ts_end = s_to_ns(60*60*24*365);
  tw_opt_add(app_opt);
  tw_init(&argc, &argv);

  if (!conf_file_name[0]) {
    fprintf(stderr, "Expected \"codes-config\" option, please see --help.\n");
    MPI_Finalize();
    return 1;
  }

  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
  
  if (configuration_load(conf_file_name, MPI_COMM_WORLD, &config)) {
    fprintf(stderr, "Error loading config file %s.\n", conf_file_name);
    MPI_Finalize();
    return 1;
  }

  model_net_register();
  node_add_lp_type();
  codes_mapping_setup();
  net_ids = model_net_configure(&num_nets);
  assert(num_nets==1);
  net_id = *net_ids;
  free(net_ids);
  if(net_id != TORUS) {
    printf("\n This is written to simulate torus networks.");
    MPI_Finalize();
    return 0;
  }
  num_nodes = codes_mapping_get_lp_count(group_name, 0, "node", NULL, 1);
  configuration_get_value_int(&config, param_group_nm, num_reqs_key, NULL, &num_reqs);
  configuration_get_value_int(&config, param_group_nm, payload_sz_key, NULL, &payload_sz);
  configuration_get_value_int(&config, param_group_nm, num_messages_key, NULL, &num_messages);
  
  tw_run();
  
  model_net_report_stats(net_id);
  
  tw_end();
  free_node_mappings();

  return 0;
}
Example #6
0
static void node_collective_init(nodes_state * s,
                                 tw_bf * bf,
                                 nodes_message * msg,
                                 tw_lp * lp)
{
    tw_event * e_new;
    tw_lpid parent_nic_id;
    tw_stime xfer_to_nic_time;
    nodes_message * msg_new;
    int num_lps;

    msg->saved_collective_init_time = s->collective_init_time;
    s->collective_init_time = tw_now(lp);
    s->origin_svr = msg->sender_svr;

    if(s->is_leaf)
    {
        //printf("\n LP %ld sending message to parent %ld ", s->node_id, s->parent_node_id);
        /* get the global LP ID of the parent node */
        // TODO: be annotation-aware
        codes_mapping_get_lp_info(lp->gid, grp_name, &mapping_grp_id, NULL,
                                  &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
        num_lps = codes_mapping_get_lp_count(grp_name, 1, LP_CONFIG_NM,
                                             NULL, 1);
        codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, NULL, 1,
                                s->parent_node_id/num_lps, (s->parent_node_id % num_lps),
                                &parent_nic_id);

        /* send a message to the parent that the LP has entered the collective operation */
        xfer_to_nic_time = g_tw_lookahead + LEVEL_DELAY;
        //e_new = codes_event_new(parent_nic_id, xfer_to_nic_time, lp);
        void* m_data;
        e_new = model_net_method_event_new(parent_nic_id, xfer_to_nic_time,
                                           lp, TORUS, (void**)&msg_new, (void**)&m_data);

        memcpy(msg_new, msg, sizeof(nodes_message));
        if (msg->remote_event_size_bytes) {
            memcpy(m_data, model_net_method_get_edata(TORUS, msg),
                   msg->remote_event_size_bytes);
        }

        msg_new->type = T_COLLECTIVE_FAN_IN;
        msg_new->sender_node = s->node_id;

        tw_event_send(e_new);
    }
    return;
}
Example #7
0
void lsm_configure(void)
{
    /* check and see if any lsm LPs are being used - otherwise,
     * skip the config */
    if (0 == codes_mapping_get_lp_count(NULL, 0, LSM_NAME, NULL, 1))
        return;

    anno_map = codes_mapping_get_lp_anno_map(LSM_NAME);
    assert(anno_map);
    models_anno = (disk_model_t*)malloc(anno_map->num_annos * sizeof(*models_anno));

    // read the configuration for unannotated entries
    if (anno_map->has_unanno_lp > 0){
        read_config(&config, NULL, &model_unanno);
    }

    for (int i = 0; i < anno_map->num_annos; i++){
        char const * anno = anno_map->annotations[i].ptr;
        read_config(&config, anno, &models_anno[i]);
    }
}
int main(
    int argc,
    char **argv)
{
    int num_nets;
    int *net_ids;
    g_tw_ts_end = s_to_ns(60*60*24*365); /* one year, in nsecs */

    tw_opt_add(app_opt);
    tw_init(&argc, &argv);

    if(argc < 2)
    {
	    printf("\n Usage: mpirun <args> --sync=[1,3] -- mapping_file_name.conf (optional --nkp) ");
	    MPI_Finalize();
	    return 0;
    }

    configuration_load(argv[2], MPI_COMM_WORLD, &config);

    model_net_register();
    svr_add_lp_type();
    
    codes_mapping_setup();
    
    net_ids = model_net_configure(&num_nets);
    assert(num_nets==1);
    net_id = *net_ids;
    free(net_ids);

    assert(net_id == SIMPLENET);
    assert(NUM_SERVERS == codes_mapping_get_lp_count("MODELNET_GRP", 0,
                "server", NULL, 1));

    tw_run();

    tw_end();
    return prog_rtn;
}
int main(int argc, char *argv[]) {
	//printf("In main\n");
	g_tw_ts_end = s_to_ns(60 * 60 * 24 * 365); /* one year, in nsecs */

	/* ROSS initialization function calls */
	tw_opt_add(app_opt); /* add user-defined args */
	/* initialize ROSS and parse args. NOTE: tw_init calls MPI_Init */
	tw_init(&argc, &argv);

	if (!conf_file_name[0]) {
		tw_error(TW_LOC,
				"Expected \"codes-config\" option, please see --help.\n");
		return 1;
	}

	/* loading the config file into the codes-mapping utility, giving us the
	 * parsed config object in return.
	 * "config" is a global var defined by codes-mapping */
	if (configuration_load(conf_file_name, MPI_COMM_WORLD, &config)) {
		tw_error(TW_LOC, "Error loading config file %s.\n", conf_file_name);
		return 1;
	}
	lsm_register();
	//lsm_configure();
	/* register model-net LPs with ROSS */
	model_net_register();

	/* register the user LPs */
	node_register();
	forwarder_register();

	/* setup the LPs and codes_mapping structures */
	codes_mapping_setup();

	/* setup the globals */
	int rc = configuration_get_value_int(&config, "server_pings", "num_reqs",
			NULL, &num_reqs);
	if (rc != 0)
		tw_error(TW_LOC, "unable to read server_pings:num_reqs");
	int payload_sz_d;
	rc = configuration_get_value_int(&config, "server_pings", "payload_sz",
			NULL, &payload_sz_d);
	if (rc != 0)
		tw_error(TW_LOC, "unable to read server_pings:payload_sz");
	payload_sz = (uint64_t) payload_sz_d;

	/* get the counts for the client and svr clusters */
	num_client_nodes = codes_mapping_get_lp_count("client_CLUSTER", 0, "node",
			NULL, 1);
	num_svr_nodes = codes_mapping_get_lp_count("svr_CLUSTER", 0, "node", NULL,
			1);
	num_burst_buffer_nodes = codes_mapping_get_lp_count("bb_CLUSTER", 0, "node",
			NULL, 1);
	num_storage_nodes = codes_mapping_get_lp_count("storage_CLUSTER", 0, "node",
			NULL, 1);
	num_client_forwarders = codes_mapping_get_lp_count("client_FORWARDERS", 0,
			"forwarder", NULL, 1);
	num_svr_forwarders = codes_mapping_get_lp_count("svr_FORWARDERS", 0,
			"forwarder", NULL, 1);
	num_burst_buffer_forwarders = codes_mapping_get_lp_count("bb_FORWARDERS", 0,
			"forwarder", NULL, 1);
	num_storage_forwarders = codes_mapping_get_lp_count("storage_FORWARDERS", 0,
			"forwarder", NULL, 1);

	/* Setup the model-net parameters specified in the global config object,
	 * returned are the identifier(s) for the network type.
	 * 1 ID  -> all the same modelnet model
	 * 2 IDs -> clusters are the first id, forwarding network the second
	 * 3 IDs -> client is first, svr and bb second and forwarding network the third
	 * 4 IDs -> cluster client is the first, svr is the second, burst buffer the third and forwarding network the last
	 *          */
	int num_nets;
	int *net_ids = model_net_configure(&num_nets);
	assert(num_nets <= 5);
	if (num_nets == 1) {
		net_id_client = net_ids[0];
		net_id_svr = net_ids[0];
		net_id_bb = net_ids[0];
		net_id_storage = net_ids[0];
		net_id_forwarding = net_ids[0];
	} else if (num_nets == 2) {
		net_id_client = net_ids[0];
		net_id_svr = net_ids[0];
		net_id_bb = net_ids[0];
		net_id_storage = net_ids[0];
		net_id_forwarding = net_ids[1];
	} else if (num_nets == 3) {
		net_id_client = net_ids[0];
		net_id_svr = net_ids[1];
		net_id_bb = net_ids[1];
		net_id_storage = net_ids[1];
		net_id_forwarding = net_ids[2];
	} else if (num_nets == 4) {
		net_id_client = net_ids[0];
		net_id_svr = net_ids[1];
		net_id_bb = net_ids[2];
		net_id_storage = net_ids[2];
		net_id_forwarding = net_ids[3];
	} else {
		net_id_client = net_ids[0];
		net_id_svr = net_ids[1];
		net_id_bb = net_ids[2];
		net_id_storage = net_ids[3];
		net_id_forwarding = net_ids[4];
	}
	free(net_ids);

	configuration_get_value_int(&config, param_group_nm, num_reqs_key, NULL,
			&num_reqs);
	configuration_get_value_int(&config, param_group_nm, payload_sz_key, NULL,
			(int *) &payload_sz);
	configuration_get_value_int(&config, param_group_nm, pvfs_file_sz_key, NULL,
			&pvfs_file_sz); /*Sughosh: added for pvfsfs*/
	configuration_get_value_int(&config, param_group_nm, bb_file_size_key, NULL,
			&bb_file_sz); /*Tony: added for bb*/
	configuration_get_value_int(&config, param_group_nm, bb_capacity_key, NULL,
			&burst_buffer_max_capacity); /*Tony: added for bb*/

	/* begin simulation */
	model_net_report_stats(net_id);
	tw_run();

	tw_end();

	return 0;
}
int main( int argc, char** argv )
{
  int rank, nprocs;
  int num_nets;
  int* net_ids;

  g_tw_ts_end = s_to_ns(60*60*24*365); /* one year, in nsecs */

  workload_type[0]='\0';
  tw_opt_add(app_opt);
  tw_init(&argc, &argv);

  if(strlen(workload_file) == 0)
    {
	if(tw_ismaster())
		printf("\n Usage: mpirun -np n ./codes-nw-test --sync=1/2/3 --workload_type=type --workload_file=workload-file-name");
	tw_end();
	return -1;
    }

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

   configuration_load(argv[2], MPI_COMM_WORLD, &config);

   nw_add_lp_type();

   codes_mapping_setup();

   num_net_lps = codes_mapping_get_lp_count("MODELNET_GRP", 0, "nw-lp", NULL, 0);
   
    tw_run();

    long long total_bytes_sent, total_bytes_recvd;
    double avg_run_time;
    double avg_comm_run_time;
    double avg_col_run_time;
    double total_avg_send_time;
    double total_avg_wait_time;
    double total_avg_recv_time;
    double total_avg_col_time;
    double total_avg_comp_time;
    long overall_sends, overall_recvs, overall_waits, overall_cols;
	
    MPI_Reduce(&num_bytes_sent, &total_bytes_sent, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
    MPI_Reduce(&num_bytes_recvd, &total_bytes_recvd, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce(&avg_time, &avg_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);

   MPI_Reduce(&avg_recv_time, &total_avg_recv_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce(&avg_comm_time, &avg_comm_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce(&avg_col_time, &avg_col_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
  MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce(&avg_compute_time, &total_avg_comp_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce(&total_sends, &overall_sends, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce(&total_recvs, &overall_recvs, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce(&total_waits, &overall_waits, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce(&total_collectives, &overall_cols, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);

   if(!g_tw_mynode)
	printf("\n Total bytes sent %lld recvd %lld \n avg runtime %lf \n avg comm time %lf avg compute time %lf \n avg collective time %lf avg send time %lf \n avg recv time %lf \n avg wait time %lf \n total sends %ld total recvs %ld total waits %ld total collectives %ld ", total_bytes_sent, total_bytes_recvd, 
			avg_run_time/num_net_lps,
			avg_comm_run_time/num_net_lps,
			total_avg_comp_time/num_net_lps,
			total_avg_col_time/num_net_lps,
			total_avg_send_time/num_net_lps,
			total_avg_recv_time/num_net_lps,
			total_avg_wait_time/num_net_lps,
			overall_sends, overall_recvs, overall_waits, overall_cols);
   tw_end();
  
  return 0;
}
Example #11
0
static void node_collective_fan_in(nodes_state * s,
                                   tw_bf * bf,
                                   nodes_message * msg,
                                   tw_lp * lp)
{
    int i;
    s->num_fan_nodes++;

    // TODO: be annotation-aware
    codes_mapping_get_lp_info(lp->gid, grp_name, &mapping_grp_id, NULL,
                              &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
    int num_lps = codes_mapping_get_lp_count(grp_name, 1, LP_CONFIG_NM,
                  NULL, 1);

    tw_event* e_new;
    nodes_message * msg_new;
    tw_stime xfer_to_nic_time;

    bf->c1 = 0;
    bf->c2 = 0;

    /* if the number of fanned in nodes have completed at the current node then signal the parent */
    if((s->num_fan_nodes == s->num_children) && !s->is_root)
    {
        bf->c1 = 1;
        msg->saved_fan_nodes = s->num_fan_nodes-1;
        s->num_fan_nodes = 0;
        tw_lpid parent_nic_id;
        xfer_to_nic_time = g_tw_lookahead + LEVEL_DELAY;

        /* get the global LP ID of the parent node */
        codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, NULL, 1,
                                s->parent_node_id/num_lps, (s->parent_node_id % num_lps),
                                &parent_nic_id);

        /* send a message to the parent that the LP has entered the collective operation */
        //e_new = codes_event_new(parent_nic_id, xfer_to_nic_time, lp);
        //msg_new = tw_event_data(e_new);
        void * m_data;
        e_new = model_net_method_event_new(parent_nic_id,
                                           xfer_to_nic_time,
                                           lp, TORUS, (void**)&msg_new, &m_data);

        memcpy(msg_new, msg, sizeof(nodes_message));
        msg_new->type = T_COLLECTIVE_FAN_IN;
        msg_new->sender_node = s->node_id;

        if (msg->remote_event_size_bytes) {
            memcpy(m_data, model_net_method_get_edata(TORUS, msg),
                   msg->remote_event_size_bytes);
        }

        tw_event_send(e_new);
    }

    /* root node starts off with the fan-out phase */
    if(s->is_root && (s->num_fan_nodes == s->num_children))
    {
        bf->c2 = 1;
        msg->saved_fan_nodes = s->num_fan_nodes-1;
        s->num_fan_nodes = 0;
        send_remote_event(s, bf, msg, lp);

        for( i = 0; i < s->num_children; i++ )
        {
            tw_lpid child_nic_id;
            /* Do some computation and fan out immediate child nodes from the collective */
            xfer_to_nic_time = g_tw_lookahead + COLLECTIVE_COMPUTATION_DELAY + LEVEL_DELAY + tw_rand_exponential(lp->rng, (double)LEVEL_DELAY/50);

            /* get global LP ID of the child node */
            codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, NULL, 1,
                                    s->children[i]/num_lps, (s->children[i] % num_lps),
                                    &child_nic_id);
            //e_new = codes_event_new(child_nic_id, xfer_to_nic_time, lp);

            //msg_new = tw_event_data(e_new);
            void * m_data;
            e_new = model_net_method_event_new(child_nic_id,
                                               xfer_to_nic_time,
                                               lp, TORUS, (void**)&msg_new, &m_data);

            memcpy(msg_new, msg, sizeof(nodes_message));
            if (msg->remote_event_size_bytes) {
                memcpy(m_data, model_net_method_get_edata(TORUS, msg),
                       msg->remote_event_size_bytes);
            }

            msg_new->type = T_COLLECTIVE_FAN_OUT;
            msg_new->sender_node = s->node_id;

            tw_event_send(e_new);
        }
    }
}
int main(
    int argc,
    char **argv)
{
    int nprocs;
    int rank;
    int num_nets;
    int *net_ids;
    char* anno;

    lp_io_handle handle;

    tw_opt_add(app_opt);
    tw_init(&argc, &argv);
    offset = 1;

    if(argc < 2)
    {
            printf("\n Usage: mpirun <args> --sync=2/3 mapping_file_name.conf (optional --nkp) ");
            MPI_Finalize();
            return 0;
    }

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

    configuration_load(argv[2], MPI_COMM_WORLD, &config);

    model_net_register();
    svr_add_lp_type();

    codes_mapping_setup();

    net_ids = model_net_configure(&num_nets);
    assert(num_nets==1);
    net_id = *net_ids;
    free(net_ids);

    if(net_id != DRAGONFLY)
    {
	printf("\n The test works with dragonfly model configuration only! ");
        MPI_Finalize();
        return 0;
    }
    num_servers_per_rep = codes_mapping_get_lp_count("MODELNET_GRP", 1, "server",
            NULL, 1);
    configuration_get_value_int(&config, "PARAMS", "num_routers", anno, &num_routers_per_grp);
    
    num_groups = (num_routers_per_grp * (num_routers_per_grp/2) + 1);
    num_nodes = num_groups * num_routers_per_grp * (num_routers_per_grp / 2);
    num_nodes_per_grp = num_routers_per_grp * (num_routers_per_grp / 2);

    if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_WORLD) < 0)
    {
        return(-1);
    }

    tw_run();
    model_net_report_stats(net_id);

    if(lp_io_flush(handle, MPI_COMM_WORLD) < 0)
    {
        return(-1);
    }

    tw_end();
    return 0;
}