static void action_send(const char *const *action)
{
int to = atoi(action[2]);
double size=parse_double(action[3]);
double clock = smpi_process_simulated_elapsed();
#ifdef HAVE_TRACING
int rank = smpi_comm_rank(MPI_COMM_WORLD);
TRACE_smpi_computing_out(rank);
int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
TRACE_smpi_ptp_in(rank, rank, dst_traced, __FUNCTION__);
TRACE_smpi_send(rank, rank, dst_traced);
#endif
smpi_mpi_send(NULL, size, MPI_BYTE, to , 0, MPI_COMM_WORLD);
if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
char *name = xbt_str_join_array(action, " ");
XBT_VERB("%s %f", name, smpi_process_simulated_elapsed()-clock);
free(name);
}
#ifdef HAVE_TRACING
TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
TRACE_smpi_computing_in(rank);
#endif
}
static void action_Irecv(const char *const *action)
{
int from = atoi(action[2]);
double size=parse_double(action[3]);
double clock = smpi_process_simulated_elapsed();
MPI_Request request;
smpi_replay_globals_t globals =
(smpi_replay_globals_t) smpi_process_get_user_data();
#ifdef HAVE_TRACING
int rank = smpi_comm_rank(MPI_COMM_WORLD);
int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__);
#endif
request = smpi_mpi_irecv(NULL, size, MPI_BYTE, from, 0, MPI_COMM_WORLD);
#ifdef HAVE_TRACING
TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
request->recv = 1;
#endif
xbt_dynar_push(globals->irecvs,&request);
//TODO do the asynchronous cleanup
if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
char *name = xbt_str_join_array(action, " ");
XBT_VERB("%s %f", name, smpi_process_simulated_elapsed()-clock);
free(name);
}
}
static void action_compute(const char *const *action)
{
double clock = smpi_process_simulated_elapsed();
smpi_execute_flops(parse_double(action[2]));
if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
char *name = xbt_str_join_array(action, " ");
XBT_VERB("%s %f", name, smpi_process_simulated_elapsed()-clock);
free(name);
}
}
static void action_Isend(const char *const *action)
{
int to = atoi(action[2]);
double size=parse_double(action[3]);
double clock = smpi_process_simulated_elapsed();
MPI_Request request;
if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
#ifdef HAVE_TRACING
int rank = smpi_comm_rank(MPI_COMM_WORLD);
TRACE_smpi_computing_out(rank);
int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
TRACE_smpi_ptp_in(rank, rank, dst_traced, __FUNCTION__);
TRACE_smpi_send(rank, rank, dst_traced);
#endif
request = smpi_mpi_isend(NULL, size, MPI_CURRENT_TYPE, to, 0,MPI_COMM_WORLD);
#ifdef HAVE_TRACING
TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
request->send = 1;
TRACE_smpi_computing_in(rank);
#endif
xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
log_timed_action (action, clock);
}
static void action_test(const char *const *action){
CHECK_ACTION_PARAMS(action, 0, 0);
double clock = smpi_process_simulated_elapsed();
MPI_Request request;
MPI_Status status;
int flag = TRUE;
request = xbt_dynar_pop_as(get_reqq_self(),MPI_Request);
//if request is null here, this may mean that a previous test has succeeded
//Different times in traced application and replayed version may lead to this
//In this case, ignore the extra calls.
if(request){
int rank = smpi_process_index();
instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
extra->type=TRACING_TEST;
TRACE_smpi_testing_in(rank, extra);
flag = smpi_mpi_test(&request, &status);
XBT_DEBUG("MPI_Test result: %d", flag);
/* push back request in dynar to be caught by a subsequent wait. if the test
* did succeed, the request is now NULL.
*/
xbt_dynar_push_as(get_reqq_self(),MPI_Request, request);
TRACE_smpi_testing_out(rank);
}
log_timed_action (action, clock);
}
static void log_timed_action (const char *const *action, double clock){
if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
char *name = xbt_str_join_array(action, " ");
XBT_VERB("%s %f", name, smpi_process_simulated_elapsed()-clock);
free(name);
}
}
static void action_compute(const char *const *action)
{
double clock = smpi_process_simulated_elapsed();
smpi_execute_flops(parse_double(action[2]));
log_timed_action (action, clock);
}
static void action_Irecv(const char *const *action)
{
int from = atoi(action[2]);
double size=parse_double(action[3]);
double clock = smpi_process_simulated_elapsed();
MPI_Request request;
smpi_replay_globals_t globals =
(smpi_replay_globals_t) smpi_process_get_user_data();
if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
#ifdef HAVE_TRACING
int rank = smpi_comm_rank(MPI_COMM_WORLD);
int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__);
#endif
request = smpi_mpi_irecv(NULL, size, MPI_CURRENT_TYPE, from, 0, MPI_COMM_WORLD);
#ifdef HAVE_TRACING
TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
request->recv = 1;
#endif
xbt_dynar_push(globals->irecvs,&request);
xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
log_timed_action (action, clock);
}
static void action_comm_size(const char *const *action)
{
double clock = smpi_process_simulated_elapsed();
communicator_size = parse_double(action[2]);
log_timed_action (action, clock);
}
static void action_Irecv(const char *const *action)
{
CHECK_ACTION_PARAMS(action, 2, 1);
int from = atoi(action[2]);
double size=parse_double(action[3]);
double clock = smpi_process_simulated_elapsed();
MPI_Request request;
if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
int rank = smpi_process_index();
int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
extra->type = TRACING_IRECV;
extra->send_size = size;
extra->src = src_traced;
extra->dst = rank;
extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE, NULL);
TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__, extra);
MPI_Status status;
//unknow size from the receiver pov
if(size==-1){
smpi_mpi_probe(from, 0, MPI_COMM_WORLD, &status);
size=status.count;
}
request = smpi_mpi_irecv(NULL, size, MPI_CURRENT_TYPE, from, 0, MPI_COMM_WORLD);
TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
request->recv = 1;
xbt_dynar_push(get_reqq_self(),&request);
log_timed_action (action, clock);
}
static void action_Isend(const char *const *action)
{
CHECK_ACTION_PARAMS(action, 2, 1);
int to = atoi(action[2]);
double size=parse_double(action[3]);
double clock = smpi_process_simulated_elapsed();
MPI_Request request;
if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
int rank = smpi_process_index();
int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
extra->type = TRACING_ISEND;
extra->send_size = size;
extra->src = rank;
extra->dst = dst_traced;
extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE, NULL);
TRACE_smpi_ptp_in(rank, rank, dst_traced, __FUNCTION__, extra);
if (!TRACE_smpi_view_internals()) {
TRACE_smpi_send(rank, rank, dst_traced, size*smpi_datatype_size(MPI_CURRENT_TYPE));
}
request = smpi_mpi_isend(NULL, size, MPI_CURRENT_TYPE, to, 0,MPI_COMM_WORLD);
TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
request->send = 1;
xbt_dynar_push(get_reqq_self(),&request);
log_timed_action (action, clock);
}
static void action_recv(const char *const *action) {
int from = atoi(action[2]);
double size=parse_double(action[3]);
double clock = smpi_process_simulated_elapsed();
MPI_Status status;
if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
#ifdef HAVE_TRACING
int rank = smpi_comm_rank(MPI_COMM_WORLD);
int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
TRACE_smpi_computing_out(rank);
TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__);
#endif
smpi_mpi_recv(NULL, size, MPI_CURRENT_TYPE, from, 0, MPI_COMM_WORLD, &status);
#ifdef HAVE_TRACING
TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
TRACE_smpi_recv(rank, src_traced, rank);
TRACE_smpi_computing_in(rank);
#endif
log_timed_action (action, clock);
}
static void action_wait(const char *const *action){
double clock = smpi_process_simulated_elapsed();
MPI_Request request;
MPI_Status status;
smpi_replay_globals_t globals =
(smpi_replay_globals_t) smpi_process_get_user_data();
xbt_assert(xbt_dynar_length(globals->irecvs),
"action wait not preceded by any irecv: %s",
xbt_str_join_array(action," "));
request = xbt_dynar_pop_as(globals->irecvs,MPI_Request);
#ifdef HAVE_TRACING
int rank = request && request->comm != MPI_COMM_NULL
? smpi_comm_rank(request->comm)
: -1;
TRACE_smpi_computing_out(rank);
MPI_Group group = smpi_comm_group(request->comm);
int src_traced = smpi_group_rank(group, request->src);
int dst_traced = smpi_group_rank(group, request->dst);
int is_wait_for_receive = request->recv;
TRACE_smpi_ptp_in(rank, src_traced, dst_traced, __FUNCTION__);
#endif
smpi_mpi_wait(&request, &status);
#ifdef HAVE_TRACING
TRACE_smpi_ptp_out(rank, src_traced, dst_traced, __FUNCTION__);
if (is_wait_for_receive) {
TRACE_smpi_recv(rank, src_traced, dst_traced);
}
TRACE_smpi_computing_in(rank);
#endif
log_timed_action (action, clock);
}
static void action_barrier(const char *const *action){
double clock = smpi_process_simulated_elapsed();
#ifdef HAVE_TRACING
int rank = smpi_comm_rank(MPI_COMM_WORLD);
TRACE_smpi_computing_out(rank);
TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
#endif
smpi_mpi_barrier(MPI_COMM_WORLD);
#ifdef HAVE_TRACING
TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
TRACE_smpi_computing_in(rank);
#endif
if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
char *name = xbt_str_join_array(action, " ");
XBT_VERB("%s %f", name, smpi_process_simulated_elapsed()-clock);
free(name);
}
}
static void action_allReduce(const char *const *action) {
double comm_size = parse_double(action[2]);
double comp_size = parse_double(action[3]);
double clock = smpi_process_simulated_elapsed();
#ifdef HAVE_TRACING
int rank = smpi_comm_rank(MPI_COMM_WORLD);
TRACE_smpi_computing_out(rank);
TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
#endif
smpi_mpi_reduce(NULL, NULL, comm_size, MPI_BYTE, MPI_OP_NULL, 0, MPI_COMM_WORLD);
smpi_execute_flops(comp_size);
smpi_mpi_bcast(NULL, comm_size, MPI_BYTE, 0, MPI_COMM_WORLD);
#ifdef HAVE_TRACING
TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
TRACE_smpi_computing_in(rank);
#endif
if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
char *name = xbt_str_join_array(action, " ");
XBT_VERB("%s %f", name, smpi_process_simulated_elapsed()-clock);
free(name);
}
}
int smpi_replay_finalize(){
double sim_time= 1.;
/* One active process will stop. Decrease the counter*/
active_processes--;
if(!active_processes){
/* Last process alive speaking */
/* end the simulated timer */
sim_time = smpi_process_simulated_elapsed();
XBT_INFO("Simulation time %g", sim_time);
_xbt_replay_action_exit();
}
return PMPI_Finalize();
}
static void action_compute(const char *const *action)
{
CHECK_ACTION_PARAMS(action, 1, 0);
double clock = smpi_process_simulated_elapsed();
double flops= parse_double(action[2]);
int rank = smpi_process_index();
instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
extra->type=TRACING_COMPUTING;
extra->comp_size=flops;
TRACE_smpi_computing_in(rank, extra);
smpi_execute_flops(flops);
TRACE_smpi_computing_out(rank);
log_timed_action (action, clock);
}
static void action_waitall(const char *const *action){
double clock = smpi_process_simulated_elapsed();
int count_requests=0;
unsigned int i=0;
count_requests=xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
if (count_requests>0) {
MPI_Request requests[count_requests];
MPI_Status status[count_requests];
/* The reqq is an array of dynars. Its index corresponds to the rank.
Thus each rank saves its own requests to the array request. */
xbt_dynar_foreach(reqq[smpi_comm_rank(MPI_COMM_WORLD)],i,requests[i]);
#ifdef HAVE_TRACING
//save information from requests
xbt_dynar_t srcs = xbt_dynar_new(sizeof(int), NULL);
xbt_dynar_t dsts = xbt_dynar_new(sizeof(int), NULL);
xbt_dynar_t recvs = xbt_dynar_new(sizeof(int), NULL);
for (i = 0; i < count_requests; i++) {
if(requests[i]){
int *asrc = xbt_new(int, 1);
int *adst = xbt_new(int, 1);
int *arecv = xbt_new(int, 1);
*asrc = requests[i]->src;
*adst = requests[i]->dst;
*arecv = requests[i]->recv;
xbt_dynar_insert_at(srcs, i, asrc);
xbt_dynar_insert_at(dsts, i, adst);
xbt_dynar_insert_at(recvs, i, arecv);
xbt_free(asrc);
xbt_free(adst);
xbt_free(arecv);
}else {
int *t = xbt_new(int, 1);
xbt_dynar_insert_at(srcs, i, t);
xbt_dynar_insert_at(dsts, i, t);
xbt_dynar_insert_at(recvs, i, t);
xbt_free(t);
}
}
void smpi_sample_3(int global, const char *file, int line)
{
char *loc = sample_location(global, file, line);
local_data_t *data;
double sample, n;
xbt_assert(samples, "You did something very inconsistent, didn't you?");
data = xbt_dict_get_or_null(samples, loc);
smpi_bench_end();
if(data && data->started && data->count < data->iters) {
sample = smpi_process_simulated_elapsed();
data->sum += sample;
data->sum_pow2 += sample * sample;
n = (double)data->count;
data->mean = data->sum / n;
data->relstderr = sqrt((data->sum_pow2 / n - data->mean * data->mean) / n) / data->mean;
XBT_DEBUG("Average mean after %d steps is %f, relative standard error is %f (sample was %f)", data->count,
data->mean, data->relstderr, sample);
}
free(loc);
}
static void action_wait(const char *const *action){
CHECK_ACTION_PARAMS(action, 0, 0);
double clock = smpi_process_simulated_elapsed();
MPI_Request request;
MPI_Status status;
xbt_assert(xbt_dynar_length(get_reqq_self()),
"action wait not preceded by any irecv or isend: %s",
xbt_str_join_array(action," "));
request = xbt_dynar_pop_as(get_reqq_self(),MPI_Request);
if (!request){
/* Assuming that the trace is well formed, this mean the comm might have
* been caught by a MPI_test. Then just return.
*/
return;
}
int rank = request->comm != MPI_COMM_NULL
? smpi_comm_rank(request->comm)
: -1;
MPI_Group group = smpi_comm_group(request->comm);
int src_traced = smpi_group_rank(group, request->src);
int dst_traced = smpi_group_rank(group, request->dst);
int is_wait_for_receive = request->recv;
instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
extra->type = TRACING_WAIT;
TRACE_smpi_ptp_in(rank, src_traced, dst_traced, __FUNCTION__, extra);
smpi_mpi_wait(&request, &status);
TRACE_smpi_ptp_out(rank, src_traced, dst_traced, __FUNCTION__);
if (is_wait_for_receive)
TRACE_smpi_recv(rank, src_traced, dst_traced);
log_timed_action (action, clock);
}
double clock = smpi_process_simulated_elapsed();
log_timed_action (action, clock);
}
static void action_comm_dup(const char *const *action)
{
double clock = smpi_process_simulated_elapsed();
log_timed_action (action, clock);
}
static void action_compute(const char *const *action)
{
CHECK_ACTION_PARAMS(action, 1, 0)
double clock = smpi_process_simulated_elapsed();
double flops= parse_double(action[2]);
int rank = smpi_process_index();
instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
extra->type=TRACING_COMPUTING;
extra->comp_size=flops;
TRACE_smpi_computing_in(rank, extra);
smpi_execute_flops(flops);
TRACE_smpi_computing_out(rank);
log_timed_action (action, clock);
}
static void action_send(const char *const *action)
{
static void action_comm_dup(const char *const *action)
{
double clock = smpi_process_simulated_elapsed();
log_timed_action (action, clock);
}
