int main( int argc, char** argv ) {
struct log_context* logctx = NULL;
int rc = 0;
if( strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0 ) {
fprintf(stderr, "Log test program\n");
trace_usage();
exit(0);
}
rc = trace_begin( &logctx );
if( rc != 0 ) {
fprintf(stderr, "trace_begin rc = %d\n", rc );
exit(1);
}
log_argv( logctx, argc, argv );
// compress
rc = log_rollover( logctx );
if( rc != 0 ) {
fprintf(stderr, "log_rollover rc = %d\n", rc );
exit(1);
}
log_argv( logctx, argc, argv );
// compress, again
rc = log_rollover( logctx );
if( rc != 0 ) {
fprintf(stderr, "log_rollover rc = %d\n", rc );
exit(1);
}
// where are the compressed logs?
print_compressed_log_paths( logctx );
// clean up
trace_end( &logctx );
return 0;
}
int main(int argc, char **argv) {
double sum_total_timer, total_timer = 0.0;
double sum_gather_timer, gather_timer = 0.0;
double sum_mpi_timer, mpi_timer = 0.0;
double curr_time;
double output_time;
double dt = 0.0;
double local_max_norm = 0.1;
double max_norm = 0;
int steps;
int* fish_off;
int* n_fish_split;
MPI_Init (&argc, &argv);
#ifdef TRACE_WITH_VAMPIR
VT_symdef(TRACE_LOCAL_COMP, "Local computation", "Computation");
VT_symdef(TRACE_FISH_GATHER, "Gathering to 0", "Communication");
VT_symdef(TRACE_MAX_NORM, "Collecting max norm", "Communication");
VT_symdef(TRACE_OUTPUT, "Output", "Output");
#endif
MPI_Comm_size (comm, &n_proc);
MPI_Comm_rank (comm, &rank);
make_fishtype (&fishtype);
get_options(argc, argv);
srand48(clock());
//MPI_Allreduce (&local_max_norm, &max_norm, 1, MPI_DOUBLE, MPI_MAX, comm);
//printf("local_max_norm = %g, max_norm = %g\n", local_max_norm, max_norm);
#ifdef TRACE_WITH_VAMPIR
VT_traceoff();
#endif
if (output_filename) {
outputp = 1;
if (0 == rank) {
output_fp = fopen(output_filename, "w");
if (output_fp == NULL) {
printf("Could not open %s for output\n", output_filename);
exit(1);
}
fprintf(output_fp, "n_fish: %d\n", n_fish);
}
}
fish_off = malloc ( (n_proc+1) * sizeof(int) );
n_fish_split = malloc ( (n_proc) * sizeof(int) );
//split each fish to different processors.
//fish_off: offset index of the fish in that processor
//n_fish_split is the # of fish in each processor
//ALL FUNCTIONALITY OF split_fish SHOULD BE DONE AFTER init_fish
//split_fish (n_proc, fish_off, n_fish_split);
//n_local_fish = n_fish_split[rank];
/*
All fish are generated on proc 0 to ensure same random numbers.
(Yes, the circle case could be parallelized. Feel free to
do it.)
*/
//split physical box sizes
row = (int)sqrt((double)n_proc);
column = n_proc/row;
double rowSep = WALL_SEP/row;
double columnSep = WALL_SEP/column;
int rowIndex = rank / column;
int columnIndex = rank % column;
topBound = rowSep * rowIndex;
bottomBound = topBound + rowSep;
leftBound = columnSep * columnIndex;
rightBound = leftBound + columnSep;
assert(n_proc % row == 0);
// Add n_proc # of arrays each holding ID of local fishes
fish_t fishProc[n_proc][n_fish];
int n_fish_proc[n_proc];
int k;
for (k = 0; k < n_proc; k++) n_fish_proc[k] = 0;
//////////////////////////////////
init_fish (rank, fish_off, n_fish_split, row, column, fishProc, n_fish_proc);
// distribute initial conditions to all processes
if (rank == 0) {
local_fish = fishProc[0];
n_local_fish = n_fish_proc[0];
// Functionality of MPI_Scatterv is done here with Isends
//MPI_Request request[n_proc-1];
int mesTag = 0;
MPI_Request *req;
for (k = 1; k < n_proc; ++k) {
//printf("n_fish_proc[%d], %d\n", k, n_fish_proc[k]);
MPI_Isend(fishProc[k], n_fish_proc[k], fishtype, k, mesTag, comm, req);
}
} else {
MPI_Status status;
// Processors of rank != 0 receives.
MPI_Recv( local_fish, n_fish, fishtype, 0, MPI_ANY_TAG, comm, &status);
MPI_Get_count(&status, fishtype, &n_local_fish);
}
printf("rank[%d], n_local_fish = %d\n", rank, n_local_fish);
///*
//MPI_Scatterv (fish, n_fish_split, fish_off, fishtype,
// local_fish, n_local_fish, fishtype,
// 0, comm);
//*/
#ifdef TRACE_WITH_VAMPIR
tracingp = 1;
VT_traceon();
#endif
start_mpi_timer(&total_timer);
for (output_time = 0.0, curr_time = 0.0, steps = 0;
curr_time <= end_time && steps < max_steps;
curr_time += dt, ++steps) {
#ifdef TRACE_WITH_VAMPIR
if (steps >= STEPS_TO_TRACE) {
tracingp = 0; VT_traceoff();
}
#endif
trace_begin(TRACE_FISH_GATHER);
start_mpi_timer (&gather_timer);
start_mpi_timer (&mpi_timer);
/*
Pull in all the fish. Obviously, this is not a good idea.
You will be greatly expanding this one line...
However, feel free to waste memory when producing output.
If you're dumping fish to a file, go ahead and do an
Allgatherv _in the output steps_ if you want. Or you could
pipeline dumping the fish.
MPI_Allgatherv (local_fish, n_local_fish, fishtype,
fish, n_fish_split, fish_off, fishtype, comm);
*/
//MPI_Request* sendReq, recvReq;
// Set aside buffer for fish received from other processes.
/*
for (j = 0; j < NUM_NEIGHBOR; ++j) {
//FIXME: which neighbors does not exist?
if (rankNeighbor[j] >= 0) {
MPI_Isend(local_fish, n_local_fish, fishtype, rankNeighbor[j], MPI_ANY_TAG, comm, &sendReqArray);
MPI_Irecv(impact_fish, n_fish, fishtype, rankNeighbor[NUM_NEIGHBOR - j], MPI_ANY_TAG, comm, &sendReqArray);
MPI_Wait(recvReq, MPI_STATUS_IGNORE);
interact_fish_mpi(local_fish, n_local_fish, impact_fish, sizeof(impact_fish));
}
}
*/
// get migrate fish
// send migrate fish
// receive migrate fish
// update local fish
// get impact fish
// send impact fish
// receive impact fish
// interact impact fish
// interact local fish
// move
MPI_Request sendReqArray[NUM_NEIGHBOR];
MPI_Request recvReqArray[NUM_NEIGHBOR];
fish_t receive_impact_fish[NUM_NEIGHBOR][n_fish];
int n_receive_impact_fish[NUM_NEIGHBOR];
fish_t receive_migrate_fish[NUM_NEIGHBOR][n_fish];
int n_receive_migrate_fish[NUM_NEIGHBOR];
int n_send_impact_fish[NUM_NEIGHBOR];
fish_t* send_impact_fish[NUM_NEIGHBOR];
int n_send_migrate_fish[NUM_NEIGHBOR];
fish_t* send_migrate_fish[NUM_NEIGHBOR];
get_interacting_fish( local_fish, n_local_fish, send_migrate_fish, n_send_migrate_fish, 1);
int tmp;
for (tmp = 0; tmp < NUM_NEIGHBOR; tmp++) {
printf("rank[%d], iter[%d] ------- get [%d] migrate fish for neig[%d]. \n", rank, iter, n_send_migrate_fish[tmp], tmp);
}
Isend_receive_fish(send_migrate_fish, n_send_migrate_fish, receive_migrate_fish, n_fish, sendReqArray, recvReqArray);
wait_for_fish(recvReqArray, n_receive_migrate_fish);
// FIXME: Have not implement update on local fish.
//update_local_fish();
get_interacting_fish(local_fish, n_local_fish, send_impact_fish, n_send_impact_fish, 0);
for (tmp = 0; tmp < NUM_NEIGHBOR; tmp++) {
printf("rank[%d], iter[%d] ------- get [%d] impact fish for neig[%d]. \n", rank, iter, n_send_impact_fish[tmp], tmp);
}
Isend_receive_fish(send_impact_fish, n_send_impact_fish, receive_impact_fish, n_fish, sendReqArray, recvReqArray);
wait_for_fish(recvReqArray, n_receive_impact_fish);
int index;
for (index = 0; index < NUM_NEIGHBOR; index++) {
if (n_receive_impact_fish[index] > 0) {
interact_fish_mpi(local_fish, n_local_fish, receive_impact_fish[index], n_receive_impact_fish[index]);
}
}
//*/
// make sure we are sending and receiving the same # msg.
//assert(dbg == 0);
// While waiting, interact with fish in its own pocket first
printf("rank[%d], iter[%d] ------- interact [%d] local fishes\n", rank, iter, n_local_fish);
interact_fish_mpi(local_fish, n_local_fish, local_fish, n_local_fish);
printf("rank[%d], iter[%d] ------- finished interact local fish\n", rank, iter);
stop_mpi_timer (&gather_timer);
stop_mpi_timer (&mpi_timer);
trace_end(TRACE_FISH_GATHER);
/*
We only output once every output_interval time unit, at most.
Without that restriction, we can easily create a huge output
file. Printing a record for ten fish takes about 300 bytes, so
for every 1000 steps, we could dump 300K of info. Now scale the
number of fish by 1000...
*/
trace_begin(TRACE_OUTPUT);
if (outputp && curr_time >= output_time) {
if (0 == rank)
output_fish (output_fp, curr_time, dt, fish, n_fish);
output_time = curr_time + output_interval;
}
trace_end(TRACE_OUTPUT);
trace_begin (TRACE_LOCAL_COMP);
//interact_fish (local_fish, n_local_fish, fish, n_fish);
local_max_norm = compute_norm (local_fish, n_local_fish);
trace_end (TRACE_LOCAL_COMP);
trace_begin (TRACE_MAX_NORM);
start_mpi_timer (&mpi_timer);
printf("rank[%d], iter[%d] ------- Allreduce max_norm, \n", rank, iter);
MPI_Allreduce (&local_max_norm, &max_norm, 1, MPI_DOUBLE, MPI_MAX, comm);
printf("rank[%d], iter[%d] ------- local_max_norm: %g, max_norm: %g\n", local_max_norm, max_norm);
stop_mpi_timer (&mpi_timer);
trace_end (TRACE_MAX_NORM);
trace_begin (TRACE_LOCAL_COMP);
dt = max_norm_change / max_norm;
dt = f_max(dt, min_dt);
dt = f_min(dt, max_dt);
printf("rank[%d], iter[%d] ------- moving [%d] local_fish, \n", rank, iter, n_local_fish);
move_fish(local_fish, n_local_fish, dt);
printf("rank[%d], iter[%d] ------- finished moving.\n", rank, iter);
trace_end (TRACE_LOCAL_COMP);
iter++;
}
stop_mpi_timer(&total_timer);
#ifdef TRACE_WITH_VAMPIR
VT_traceoff();
#endif
if (outputp) {
MPI_Allgatherv (local_fish, n_local_fish, fishtype,
fish, n_fish_split, fish_off, fishtype, comm);
if (0 == rank) {
output_fish (output_fp, curr_time, dt, fish, n_fish);
printf("\tEnded at %g (%g), %d (%d) steps\n",
curr_time, end_time, steps, max_steps);
}
}
printf("rank[%d], ------- 39, \n", rank);
MPI_Reduce (&total_timer, &sum_total_timer, 1, MPI_DOUBLE,
MPI_SUM, 0, comm);
printf("rank[%d], ------- 40, \n", rank);
MPI_Reduce (&gather_timer, &sum_gather_timer, 1, MPI_DOUBLE,
MPI_SUM, 0, comm);
printf("rank[%d], ------- 41, \n", rank);
MPI_Reduce (&mpi_timer, &sum_mpi_timer, 1, MPI_DOUBLE,
MPI_SUM, 0, comm);
printf("rank[%d], ------- 42, \n", rank);
if (0 == rank) {
printf("Number of PEs: %d\n"
"Time taken on 0: %g (avg. %g)\n"
"Time in gathers on 0: %g (avg %g)\n"
"Time in MPI on 0: %g (avg %g)\n",
n_proc,
total_timer, sum_total_timer / n_proc,
gather_timer, sum_gather_timer / n_proc,
mpi_timer, sum_mpi_timer / n_proc);
}
printf("rank[%d], ------- 43, \n", rank);
MPI_Barrier (comm);
printf("rank[%d], ------- 44, \n", rank);
MPI_Finalize ();
printf("rank[%d], ------- done!!, \n", rank);
return 0;
}
int
main (int argc, char **argv)
{
#ifdef ENABLE_TRACE
irodsOper.getattr = traced_irodsGetattr;
irodsOper.readlink = traced_irodsReadlink;
irodsOper.readdir = traced_irodsReaddir;
irodsOper.mknod = traced_irodsMknod;
irodsOper.mkdir = traced_irodsMkdir;
irodsOper.symlink = traced_irodsSymlink;
irodsOper.unlink = traced_irodsUnlink;
irodsOper.rmdir = traced_irodsRmdir;
irodsOper.rename = traced_irodsRename;
irodsOper.link = traced_irodsLink;
irodsOper.chmod = traced_irodsChmod;
irodsOper.chown = traced_irodsChown;
irodsOper.truncate = traced_irodsTruncate;
irodsOper.utimens = traced_irodsUtimens;
irodsOper.open = traced_irodsOpen;
irodsOper.read = traced_irodsRead;
irodsOper.write = traced_irodsWrite;
irodsOper.statfs = traced_irodsStatfs;
irodsOper.release = traced_irodsRelease;
irodsOper.fsync = traced_irodsFsync;
irodsOper.flush = traced_irodsFlush;
#else
irodsOper.getattr = irodsGetattr;
irodsOper.readlink = irodsReadlink;
irodsOper.readdir = irodsReaddir;
irodsOper.mknod = irodsMknod;
irodsOper.mkdir = irodsMkdir;
irodsOper.symlink = irodsSymlink;
irodsOper.unlink = irodsUnlink;
irodsOper.rmdir = irodsRmdir;
irodsOper.rename = irodsRename;
irodsOper.link = irodsLink;
irodsOper.chmod = irodsChmod;
irodsOper.chown = irodsChown;
irodsOper.truncate = irodsTruncate;
irodsOper.utimens = irodsUtimens;
irodsOper.open = irodsOpen;
irodsOper.read = irodsRead;
irodsOper.write = irodsWrite;
irodsOper.statfs = irodsStatfs;
irodsOper.release = irodsRelease;
irodsOper.fsync = irodsFsync;
irodsOper.flush = irodsFlush;
#endif // ENABLE_TRACE
int status;
rodsArguments_t myRodsArgs;
char *optStr;
int new_argc;
char** new_argv;
#ifdef __cplusplus
#ifdef ENABLE_TRACE
bzero (&irodsOper, sizeof (irodsOper));
irodsOper.getattr = traced_irodsGetattr;
irodsOper.readlink = traced_irodsReadlink;
irodsOper.readdir = traced_irodsReaddir;
irodsOper.mknod = traced_irodsMknod;
irodsOper.mkdir = traced_irodsMkdir;
irodsOper.symlink = traced_irodsSymlink;
irodsOper.unlink = traced_irodsUnlink;
irodsOper.rmdir = traced_irodsRmdir;
irodsOper.rename = traced_irodsRename;
irodsOper.link = traced_irodsLink;
irodsOper.chmod = traced_irodsChmod;
irodsOper.chown = traced_irodsChown;
irodsOper.truncate = traced_irodsTruncate;
irodsOper.utimens = traced_irodsUtimens;
irodsOper.open = traced_irodsOpen;
irodsOper.read = traced_irodsRead;
irodsOper.write = traced_irodsWrite;
irodsOper.statfs = traced_irodsStatfs;
irodsOper.release = traced_irodsRelease;
irodsOper.fsync = traced_irodsFsync;
irodsOper.flush = traced_irodsFlush;
#else // no ENABLE_TRACE
bzero (&irodsOper, sizeof (irodsOper));
irodsOper.getattr = irodsGetattr;
irodsOper.readlink = irodsReadlink;
irodsOper.readdir = irodsReaddir;
irodsOper.mknod = irodsMknod;
irodsOper.mkdir = irodsMkdir;
irodsOper.symlink = irodsSymlink;
irodsOper.unlink = irodsUnlink;
irodsOper.rmdir = irodsRmdir;
irodsOper.rename = irodsRename;
irodsOper.link = irodsLink;
irodsOper.chmod = irodsChmod;
irodsOper.chown = irodsChown;
irodsOper.truncate = irodsTruncate;
irodsOper.utimens = irodsUtimens;
irodsOper.open = irodsOpen;
irodsOper.read = irodsRead;
irodsOper.write = irodsWrite;
irodsOper.statfs = irodsStatfs;
irodsOper.release = irodsRelease;
irodsOper.fsync = irodsFsync;
irodsOper.flush = irodsFlush;
#endif // ENABLE_TRACE
#endif
status = getRodsEnv (&MyRodsEnv);
if (status < 0) {
rodsLogError(LOG_ERROR, status, "main: getRodsEnv error. ");
exit (1);
}
/* handle iRODS-FUSE specific command line options*/
status = parseFuseSpecificCmdLineOpt (argc, argv);
if (status < 0) {
printf("Use -h for help.\n");
exit (1);
}
status = makeCleanCmdLineOpt (argc, argv, &new_argc, &new_argv);
argc = new_argc;
argv = new_argv;
optStr = "hdo:";
status = parseCmdLineOpt (argc, argv, optStr, 0, &myRodsArgs);
if (status < 0) {
printf("Use -h for help.\n");
exit (1);
}
if (myRodsArgs.help==True) {
usage();
exit(0);
}
srandom((unsigned int) time(0) % getpid());
#ifdef CACHE_FILE_FOR_READ
if (setAndMkFileCacheDir () < 0) exit (1);
#endif
initPathCache ();
initIFuseDesc ();
initConn();
initFileCache();
#ifdef ENABLE_PRELOAD
// initialize preload
initPreload (&MyPreloadConfig, &MyRodsEnv, &myRodsArgs);
#endif
#ifdef ENABLE_LAZY_UPLOAD
// initialize Lazy Upload
initLazyUpload (&MyLazyUploadConfig, &MyRodsEnv, &myRodsArgs);
#endif
#ifdef ENABLE_TRACE
// start tracing
status = trace_begin( NULL );
if( status != 0 ) {
rodsLogError(LOG_ERROR, status, "main: trace_begin failed. ");
exit(1);
}
#endif
status = fuse_main (argc, argv, &irodsOper, NULL);
#ifdef ENABLE_TRACE
// stop tracing
trace_end( NULL );
#endif
/* release the preload command line options */
releaseCmdLineOpt (argc, argv);
#ifdef ENABLE_PRELOAD
// wait preload jobs
waitPreloadJobs();
#endif
#ifdef ENABLE_PRELOAD
// uninitialize preload
uninitPreload (&MyPreloadConfig);
if (MyPreloadConfig.cachePath != NULL) {
free(MyPreloadConfig.cachePath);
}
#endif
#ifdef ENABLE_LAZY_UPLOAD
// uninitialize lazy upload
uninitLazyUpload (&MyLazyUploadConfig);
if (MyLazyUploadConfig.bufferPath!=NULL) {
free(MyLazyUploadConfig.bufferPath);
}
#endif
disconnectAll ();
if (status < 0) {
exit (3);
} else {
exit(0);
}
}
int pthread_cond_timedwait(FAR pthread_cond_t *cond, FAR pthread_mutex_t *mutex, FAR const struct timespec *abstime)
{
FAR struct tcb_s *rtcb = this_task();
int ticks;
int mypid = (int)getpid();
irqstate_t int_state;
uint16_t oldstate;
int ret = OK;
int status;
trace_begin(TTRACE_TAG_TASK, "pthread_cond_timedwait");
svdbg("cond=0x%p mutex=0x%p abstime=0x%p\n", cond, mutex, abstime);
DEBUGASSERT(rtcb->waitdog == NULL);
/* pthread_cond_timedwait() is a cancellation point */
(void)enter_cancellation_point();
/* Make sure that non-NULL references were provided. */
if (!cond || !mutex) {
ret = EINVAL;
}
/* Make sure that the caller holds the mutex */
else if (mutex->pid != mypid) {
ret = EPERM;
}
/* If no wait time is provided, this function degenerates to
* the same behavior as pthread_cond_wait().
*/
else if (!abstime) {
ret = pthread_cond_wait(cond, mutex);
}
else {
/* Create a watchdog */
rtcb->waitdog = wd_create();
if (!rtcb->waitdog) {
ret = EINVAL;
} else {
svdbg("Give up mutex...\n");
/* We must disable pre-emption and interrupts here so that
* the time stays valid until the wait begins. This adds
* complexity because we assure that interrupts and
* pre-emption are re-enabled correctly.
*/
sched_lock();
int_state = irqsave();
/* Convert the timespec to clock ticks. We must disable pre-emption
* here so that this time stays valid until the wait begins.
*/
ret = clock_abstime2ticks(CLOCK_REALTIME, abstime, &ticks);
if (ret) {
/* Restore interrupts (pre-emption will be enabled when
* we fall through the if/then/else)
*/
irqrestore(int_state);
} else {
/* Check the absolute time to wait. If it is now or in the past, then
* just return with the timedout condition.
*/
if (ticks <= 0) {
/* Restore interrupts and indicate that we have already timed out.
* (pre-emption will be enabled when we fall through the
* if/then/else
*/
irqrestore(int_state);
ret = ETIMEDOUT;
} else {
/* Give up the mutex */
mutex->pid = -1;
ret = pthread_mutex_give(mutex);
if (ret != 0) {
/* Restore interrupts (pre-emption will be enabled when
* we fall through the if/then/else)
*/
irqrestore(int_state);
} else {
/* Start the watchdog */
wd_start(rtcb->waitdog, ticks, (wdentry_t)pthread_condtimedout, 2, (uint32_t)mypid, (uint32_t)SIGCONDTIMEDOUT);
/* Take the condition semaphore. Do not restore interrupts
* until we return from the wait. This is necessary to
* make sure that the watchdog timer and the condition wait
* are started atomically.
*/
status = sem_wait((sem_t *)&cond->sem);
/* Did we get the condition semaphore. */
if (status != OK) {
/* NO.. Handle the special case where the semaphore wait was
* awakened by the receipt of a signal -- presumably the
* signal posted by pthread_condtimedout().
*/
if (get_errno() == EINTR) {
sdbg("Timedout!\n");
ret = ETIMEDOUT;
} else {
ret = EINVAL;
}
}
/* The interrupts stay disabled until after we sample the errno.
* This is because when debug is enabled and the console is used
* for debug output, then the errno can be altered by interrupt
* handling! (bad)
*/
irqrestore(int_state);
}
/* Reacquire the mutex (retaining the ret). */
svdbg("Re-locking...\n");
oldstate = pthread_disable_cancel();
status = pthread_mutex_take(mutex, false);
pthread_enable_cancel(oldstate);
if (status == OK) {
mutex->pid = mypid;
} else if (ret == 0) {
ret = status;
}
}
/* Re-enable pre-emption (It is expected that interrupts
* have already been re-enabled in the above logic)
*/
sched_unlock();
}
/* We no longer need the watchdog */
wd_delete(rtcb->waitdog);
rtcb->waitdog = NULL;
}
}
svdbg("Returning %d\n", ret);
leave_cancellation_point();
trace_end(TTRACE_TAG_TASK);
return ret;
}