/* insert all items to be removed into libcircle for
* dynamic load balancing */
static void remove_libcircle(mfu_flist list, uint64_t* rmcount)
{
/* set globals for libcircle callbacks */
circle_list = list;
circle_count = 0;
/* initialize libcircle */
CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL);
/* set libcircle verbosity level */
enum CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
if (mfu_debug_level >= MFU_LOG_VERBOSE) {
// loglevel = CIRCLE_LOG_INFO;
}
CIRCLE_enable_logging(loglevel);
/* register callbacks */
CIRCLE_cb_create(&remove_create);
CIRCLE_cb_process(&remove_process);
/* run the libcircle job */
CIRCLE_begin();
CIRCLE_finalize();
/* record number of items we deleted */
*rmcount = circle_count;
return;
}
int main(void)
{
DPARALLEL_debug_level = DPARALLEL_LOG_DBG;
DPARALLEL_debug_stream = stderr;
CIRCLE_init(0, NULL, CIRCLE_DEFAULT_FLAGS);
CIRCLE_cb_process(&DPARALLEL_process);
CIRCLE_begin();
CIRCLE_finalize();
}
/*
* Main
*/
int
main (int argc, char **argv)
{
/* Locals */
char starttime_str[256];
char endtime_str[256];
char getCmd[256];
treewalk_options_st opts;
treewalk_init_opts(&opts);
/* Globals */
treewalk_init_globals();
/* Set up signal handler */
treewalk_install_signal_handlers();
/* Enable logging. */
PURGER_debug_stream = stdout;
PURGER_debug_level = PURGER_LOG_DBG;
/* Init lib circle */
int rank = CIRCLE_init(argc, argv);
if(rank < 0)
exit(1);
CIRCLE_enable_logging(CIRCLE_LOG_ERR);
PURGER_global_rank = rank;
opts.rank = rank;
/* Process command line options */
treewalk_process_options(argc,argv,&opts);
/* Init redis */
if (!benchmarking_flag && redis_init(opts.redis_hostname,opts.redis_port,opts.db_number) < 0)
{
LOG(PURGER_LOG_FATAL, "Redis error: %s", REDIS->errstr);
exit(EXIT_FAILURE);
}
/* Timing */
time(&time_started);
/* Ensure it's OK to run at this time */
if(!benchmarking_flag && treewalk_check_state(opts.rank,opts.force_flag) < 0)
exit(1);
/* Read from restart files */
if(!benchmarking_flag && opts.restart_flag)
CIRCLE_read_restarts();
/* Enable sharding */
if(!benchmarking_flag && sharded_flag)
{
sharded_count = redis_shard_init(opts.redis_hostlist,opts.redis_port,opts.db_number);
redis_command_ptr = &redis_shard_command;
}
/* Parallel section */
CIRCLE_cb_create(&add_objects);
CIRCLE_cb_process(&process_objects);
CIRCLE_begin();
/* End parallel section (well, kind of) */
/* Set state */
sprintf(getCmd,"set treewalk-rank-%d 0", rank);
if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT)<0)
{
fprintf(stderr,"Unable to %s",getCmd);
}
time(&time_finished);
struct tm * localstart = localtime( &time_started );
struct tm * localend = localtime ( &time_finished );
strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
sprintf(getCmd,"set treewalk_timestamp \"%s\"",endtime_str);
if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT) < 0)
{
fprintf(stderr,"Unable to %s",getCmd);
}
LOG(PURGER_LOG_INFO,"Files: %d\tDirs: %d\tTotal: %d\n",file_count,dir_count,file_count+dir_count);
if(!benchmarking_flag && sharded_flag)
redis_shard_finalize();
if(!benchmarking_flag)
redis_finalize();
if(rank == 0)
{
LOG(PURGER_LOG_INFO, "treewalk run started at: %s", starttime_str);
LOG(PURGER_LOG_INFO, "treewalk run completed at: %s", endtime_str);
LOG(PURGER_LOG_INFO, "treewalk total time (seconds) for this run: %f",difftime(time_finished,time_started));
}
LOG(PURGER_LOG_INFO, "\nTotal time in process_objects: %lf\n\
\tRedis commands: %lf %lf%%\n\
\tStating: %lf %lf%%\n\
\tReaddir: %lf %lf%%\n\
\tHashing: %lf %lf%%\n",
process_objects_total[1],redis_time[1],redis_time[1]/process_objects_total[1]*100.0,stat_time[1],stat_time[1]/process_objects_total[1]*100.0,readdir_time[1],readdir_time[1]/process_objects_total[1]*100.0
,hash_time[1],hash_time[1]/process_objects_total[1]*100.0);
CIRCLE_finalize();
_exit(EXIT_SUCCESS);
}
/* Set up and execute directory walk */
void mfu_flist_walk_paths(uint64_t num_paths, const char** paths,
mfu_walk_opts_t* walk_opts, mfu_flist bflist)
{
/* report walk count, time, and rate */
double start_walk = MPI_Wtime();
/* if dir_permission is set to 1 then set global variable */
SET_DIR_PERMS = 0;
if (walk_opts->dir_perms) {
SET_DIR_PERMS = 1;
}
/* if remove is set to 1 then set global variable */
REMOVE_FILES = 0;
if (walk_opts->remove) {
REMOVE_FILES = 1;
}
/* convert handle to flist_t */
flist_t* flist = (flist_t*) bflist;
/* get our rank and number of ranks in job */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* print message to user that we're starting */
if (mfu_debug_level >= MFU_LOG_VERBOSE && mfu_rank == 0) {
uint64_t i;
for (i = 0; i < num_paths; i++) {
MFU_LOG(MFU_LOG_INFO, "Walking %s", paths[i]);
}
}
/* initialize libcircle */
CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL);
/* set libcircle verbosity level */
enum CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
CIRCLE_enable_logging(loglevel);
/* TODO: check that paths is not NULL */
/* TODO: check that each path is within limits */
/* set some global variables to do the file walk */
CURRENT_NUM_DIRS = num_paths;
CURRENT_DIRS = paths;
CURRENT_LIST = flist;
/* we lookup users and groups first in case we can use
* them to filter the walk */
flist->detail = 0;
if (walk_opts->use_stat) {
flist->detail = 1;
if (flist->have_users == 0) {
mfu_flist_usrgrp_get_users(flist);
}
if (flist->have_groups == 0) {
mfu_flist_usrgrp_get_groups(flist);
}
}
/* register callbacks */
if (walk_opts->use_stat) {
/* walk directories by calling stat on every item */
CIRCLE_cb_create(&walk_stat_create);
CIRCLE_cb_process(&walk_stat_process);
// CIRCLE_cb_create(&walk_lustrestat_create);
// CIRCLE_cb_process(&walk_lustrestat_process);
}
else {
/* walk directories using file types in readdir */
CIRCLE_cb_create(&walk_readdir_create);
CIRCLE_cb_process(&walk_readdir_process);
// CIRCLE_cb_create(&walk_getdents_create);
// CIRCLE_cb_process(&walk_getdents_process);
}
/* prepare callbacks and initialize variables for reductions */
reduce_items = 0;
CIRCLE_cb_reduce_init(&reduce_init);
CIRCLE_cb_reduce_op(&reduce_exec);
CIRCLE_cb_reduce_fini(&reduce_fini);
/* run the libcircle job */
CIRCLE_begin();
CIRCLE_finalize();
/* compute global summary */
mfu_flist_summarize(bflist);
double end_walk = MPI_Wtime();
/* report walk count, time, and rate */
if (mfu_debug_level >= MFU_LOG_VERBOSE && mfu_rank == 0) {
uint64_t all_count = mfu_flist_global_size(bflist);
double time_diff = end_walk - start_walk;
double rate = 0.0;
if (time_diff > 0.0) {
rate = ((double)all_count) / time_diff;
}
MFU_LOG(MFU_LOG_INFO, "Walked %lu items in %f seconds (%f files/sec)",
all_count, time_diff, rate
);
}
/* hold procs here until summary is printed */
MPI_Barrier(MPI_COMM_WORLD);
return;
}
int
main (int argc, char **argv)
{
int index;
int c;
char *redis_hostname;
char *redis_hostlist;
int redis_port;
int time_flag = 0;
int dir_flag = 0;
int force_flag = 0;
int restart_flag = 0;
int redis_hostname_flag = 0;
benchmarking_flag = 0;
sharded_flag = 0;
int redis_port_flag = 0;
process_objects_total[2] = 0;
redis_time[2] = 0;
stat_time[2] = 0;
readdir_time[2] = 0;
redis_command_ptr = &redis_command;
/* Enable logging. */
PURGER_debug_stream = stdout;
PURGER_debug_level = PURGER_LOG_DBG;
int rank = CIRCLE_init(argc, argv);
PURGER_global_rank = rank;
opterr = 0;
while((c = getopt(argc, argv, "d:h:p:ft:l:rs:b")) != -1)
{
switch(c)
{
case 'b':
benchmarking_flag = 1;
break;
case 'd':
TOP_DIR = realpath(optarg, NULL);
if(rank == 0) LOG(PURGER_LOG_INFO,"Using %s as a root path.",TOP_DIR);
dir_flag = 1;
break;
case 'l':
PURGER_debug_level = atoi(optarg);
break;
case 'h':
redis_hostname = optarg;
redis_hostname_flag = 1;
break;
case 'p':
redis_port = atoi(optarg);
redis_port_flag = 1;
break;
case 'r':
if(rank == 0) LOG(PURGER_LOG_WARN,"You have specified to use restart files.");
restart_flag = 1;
break;
case 't':
time_flag = 1;
expire_threshold = (float)SECONDS_PER_DAY * atof(optarg);
if(rank == 0) LOG(PURGER_LOG_WARN,"Changed file expiration time to %.2f days, or %.2f seconds.",expire_threshold/(60.0*60.0*24),expire_threshold);
break;
case 's':
sharded_flag = 1;
redis_hostlist = optarg;
break;
case 'f':
force_flag = 1;
if(rank == 0) LOG(PURGER_LOG_WARN,"Warning: You have chosen to force treewalk.");
break;
case '?':
if (optopt == 'd' || optopt == 'h' || optopt == 'p' || optopt == 't' || optopt == 'l' || optopt == 's')
{
print_usage(argv);
fprintf(stderr, "Option -%c requires an argument.\n", optopt);
}
else if (isprint (optopt))
{
print_usage(argv);
fprintf(stderr, "Unknown option `-%c'.\n", optopt);
}
else
{
print_usage(argv);
fprintf(stderr,
"Unknown option character `\\x%x'.\n",
optopt);
}
exit(EXIT_FAILURE);
default:
abort();
}
}
if(restart_flag && dir_flag && !benchmarking_flag)
{
if(rank == 0) LOG(PURGER_LOG_WARN, "You have told treewalk to use both checkpoint files and a directory. You cannot combine these options.\n"
"If you use a directory, treewalk will start from scratch. If you use a checkpoint file, it will start from\n"
"from the data in the checkpoint files.\n");
exit(EXIT_FAILURE);
}
if(time_flag == 0 && !benchmarking_flag)
{
if(rank == 0) LOG(PURGER_LOG_WARN, "A file timeout value was not specified. Files older than %.2f seconds (%.2f days) will be expired.",expire_threshold,expire_threshold/(60.0*60.0*24.0));
}
if(dir_flag == 0 && !restart_flag && !benchmarking_flag)
{
print_usage(argv);
if(rank == 0) LOG(PURGER_LOG_FATAL, "You must specify a starting directory");
exit(EXIT_FAILURE);
}
if(redis_hostname_flag == 0 && !benchmarking_flag)
{
if(rank == 0) LOG(PURGER_LOG_WARN, "A hostname for redis was not specified, defaulting to localhost.");
redis_hostname = "localhost";
}
if(redis_port_flag == 0 && !benchmarking_flag)
{
if(rank == 0) LOG(PURGER_LOG_WARN, "A port number for redis was not specified, defaulting to 6379.");
redis_port = 6379;
}
for (index = optind; index < argc; index++)
LOG(PURGER_LOG_WARN, "Non-option argument %s", argv[index]);
if (!benchmarking_flag && redis_init(redis_hostname,redis_port) < 0)
{
LOG(PURGER_LOG_FATAL, "Redis error: %s", REDIS->errstr);
exit(EXIT_FAILURE);
}
time(&time_started);
if(!benchmarking_flag && treewalk_check_state(rank,force_flag) < 0)
exit(1);
if(!benchmarking_flag && restart_flag)
CIRCLE_read_restarts();
if(!benchmarking_flag && sharded_flag)
{
sharded_count = redis_shard_init(redis_hostlist,redis_port);
redis_command_ptr = &redis_shard_command;
}
CIRCLE_cb_create(&add_objects);
CIRCLE_cb_process(&process_objects);
CIRCLE_begin();
CIRCLE_finalize();
char getCmd[256];
sprintf(getCmd,"set treewalk-rank-%d 0", rank);
if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT)<0)
{
fprintf(stderr,"Unable to %s",getCmd);
exit(1);
}
time(&time_finished);
char starttime_str[256];
char endtime_str[256];
struct tm * localstart = localtime( &time_started );
struct tm * localend = localtime ( &time_finished );
strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
sprintf(getCmd,"set treewalk_timestamp \"%s\"",endtime_str);
if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT) < 0)
{
fprintf(stderr,"Unable to %s",getCmd);
}
if(rank == 0)
{
LOG(PURGER_LOG_INFO, "treewalk run started at: %s", starttime_str);
LOG(PURGER_LOG_INFO, "treewalk run completed at: %s", endtime_str);
LOG(PURGER_LOG_INFO, "treewalk total time (seconds) for this run: %f",difftime(time_finished,time_started));
LOG(PURGER_LOG_INFO, "\nTotal time in process_objects: %lf\n\
\tRedis commands: %lf %lf\n\
\tStating: %lf %lf\n\
\tReaddir: %lf %lf\n\
\tHashing: %lf %lf\n",
process_objects_total[1],redis_time[1],redis_time[1]/process_objects_total[1]*100.0,stat_time[1],stat_time[1]/process_objects_total[1]*100.0,readdir_time[1],readdir_time[1]/process_objects_total[1]*100.0
,hash_time[1],hash_time[1]/process_objects_total[1]*100.0);
}
static void mfu_flist_archive_create_libcircle(mfu_flist flist, const char* archivefile, mfu_archive_options_t* opts)
{
DTAR_flist = flist;
DTAR_user_opts = *opts;
MPI_Comm_rank(MPI_COMM_WORLD, &DTAR_rank);
/* TODO: stripe the archive file if on parallel file system */
/* init statistics */
DTAR_statistics.total_dirs = 0;
DTAR_statistics.total_files = 0;
DTAR_statistics.total_links = 0;
DTAR_statistics.total_size = 0;
DTAR_statistics.total_bytes_copied = 0;
time(&(DTAR_statistics.time_started));
DTAR_statistics.wtime_started = MPI_Wtime();
/* create the archive file */
DTAR_writer.name = archivefile;
DTAR_writer.flags = O_WRONLY | O_CREAT | O_CLOEXEC | O_LARGEFILE;
DTAR_writer.fd_tar = open(archivefile, DTAR_writer.flags, 0664);
/* get number of items in our portion of the list */
DTAR_count = mfu_flist_size(DTAR_flist);
/* allocate memory for file sizes and offsets */
uint64_t* fsizes = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
DTAR_offsets = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
/* compute local offsets for each item and total
* bytes we're contributing to the archive */
uint64_t idx;
uint64_t offset = 0;
for (idx = 0; idx < DTAR_count; idx++) {
/* assume the item takes no space */
fsizes[idx] = 0;
/* identify item type to compute its size in the archive */
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
/* directories and symlinks only need the header */
fsizes[idx] = DTAR_HDR_LENGTH;
} else if (type == MFU_TYPE_FILE) {
/* regular file requires a header, plus file content,
* and things are packed into blocks of 512 bytes */
uint64_t fsize = mfu_flist_file_get_size(DTAR_flist, idx);
/* determine whether file size is integer multiple of 512 bytes */
uint64_t rem = fsize % 512;
if (rem == 0) {
/* file content is multiple of 512 bytes, so perfect fit */
fsizes[idx] = fsize + DTAR_HDR_LENGTH;
} else {
/* TODO: check and explain this math */
fsizes[idx] = (fsize / 512 + 4) * 512;
}
}
/* increment our local offset for this item */
DTAR_offsets[idx] = offset;
offset += fsizes[idx];
}
/* execute scan to figure our global base offset in the archive file */
uint64_t global_offset = 0;
MPI_Scan(&offset, &global_offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
global_offset -= offset;
/* update offsets for each of our file to their global offset */
for (idx = 0; idx < DTAR_count; idx++) {
DTAR_offsets[idx] += global_offset;
}
/* create an archive */
struct archive* ar = archive_write_new();
archive_write_set_format_pax(ar);
int r = archive_write_open_fd(ar, DTAR_writer.fd_tar);
if (r != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
DTAR_abort(EXIT_FAILURE);
}
/* write headers for our files */
for (idx = 0; idx < DTAR_count; idx++) {
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_FILE || type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
DTAR_write_header(ar, idx, DTAR_offsets[idx]);
}
}
/* prepare libcircle */
CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL);
CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
CIRCLE_enable_logging(loglevel);
/* register callbacks */
CIRCLE_cb_create(&DTAR_enqueue_copy);
CIRCLE_cb_process(&DTAR_perform_copy);
/* run the libcircle job to copy data into archive file */
CIRCLE_begin();
CIRCLE_finalize();
/* compute total bytes copied */
uint64_t archive_size = 0;
MPI_Allreduce(&offset, &archive_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
DTAR_statistics.total_size = archive_size;
DTAR_statistics.wtime_ended = MPI_Wtime();
time(&(DTAR_statistics.time_ended));
/* print stats */
double rel_time = DTAR_statistics.wtime_ended - \
DTAR_statistics.wtime_started;
if (DTAR_rank == 0) {
char starttime_str[256];
struct tm* localstart = localtime(&(DTAR_statistics.time_started));
strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart);
char endtime_str[256];
struct tm* localend = localtime(&(DTAR_statistics.time_ended));
strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend);
/* add two 512 blocks at the end */
DTAR_statistics.total_size += 512 * 2;
/* convert bandwidth to unit */
double agg_rate_tmp;
double agg_rate = (double) DTAR_statistics.total_size / rel_time;
const char* agg_rate_units;
mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units);
MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str);
MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str);
MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size);
MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \
"(%.3" PRIu64 " bytes in %.3lf seconds)", \
agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time);
}
/* clean up */
mfu_free(&fsizes);
mfu_free(&DTAR_offsets);
/* close archive file */
archive_write_free(ar);
mfu_close(DTAR_writer.name, DTAR_writer.fd_tar);
}
int
main (int argc, char **argv)
{
int index;
int c;
char *redis_hostname;
int redis_port;
int redis_hostname_flag = 0;
int redis_port_flag = 0;
int sharded_flag = 0;
int ldap_flag = 0;
int force_flag = 0;
int db_number = 0;
sharded_count = 0;
char * redis_host_list;
redis_command_ptr = &non_sharded_redis_command;
PURGER_debug_stream = stdout;
PURGER_debug_level = PURGER_LOG_DBG;
int PURGER_global_rank = CIRCLE_init(argc, argv);
if(PURGER_global_rank < 0)
exit(1);
opterr = 0;
CIRCLE_enable_logging(CIRCLE_LOG_ERR);
while((c = getopt(argc, argv, "h:p:l:fi:s:a:o:")) != -1)
{
switch(c)
{
case 'o':
ldap_port = atoi(optarg);
break;
case 'a':
ldap_server = optarg;
ldap_flag = 1;
break;
case 'h':
redis_hostname = optarg;
redis_hostname_flag = 1;
break;
case 'i':
db_number = atoi(optarg);
break;
case 'p':
redis_port = atoi(optarg);
redis_port_flag = 1;
break;
case 'f':
force_flag = 1;
if(PURGER_global_rank == 0) LOG(PURGER_LOG_WARN,"Warning: You have chosen to force warnusers.");
break;
case 'l':
PURGER_debug_level = atoi(optarg);
break;
case 's':
redis_host_list = optarg;
sharded_flag = 1;
redis_command_ptr = &redis_blocking_shard_command;
break;
case '?':
if (PURGER_global_rank == 0 && (optopt == 'h' || optopt == 'p' || optopt == 'l'))
{
print_usage(argv);
fprintf(stderr, "Option -%c requires an argument.\n", optopt);
exit(EXIT_FAILURE);
}
else if (isprint (optopt))
{
print_usage(argv);
fprintf(stderr, "Unknown option `-%c'.\n", optopt);
exit(EXIT_FAILURE);
}
else
{
print_usage(argv);
fprintf(stderr,
"Unknown option character `\\x%x'.\n",
optopt);
exit(EXIT_FAILURE);
}
default:
abort();
}
}
if(redis_hostname_flag == 0)
{
if(PURGER_global_rank == 0) LOG(PURGER_LOG_WARN, "A hostname for redis was not specified, defaulting to localhost.");
redis_hostname = "localhost";
}
if(redis_port_flag == 0)
{
if(PURGER_global_rank == 0) LOG(PURGER_LOG_WARN, "A port number for redis was not specified, defaulting to 6379.");
redis_port = 6379;
}
for (index = optind; index < argc; index++)
LOG(PURGER_LOG_WARN, "Non-option argument %s", argv[index]);
if(redis_init(redis_hostname, redis_port, db_number) < 0)
{
LOG(PURGER_LOG_ERR,"Unable to initialize redis.");
exit(0);
}
if(sharded_flag)
{
LOG(PURGER_LOG_INFO,"Sharding enabled.");
sharded_count = redis_shard_init(redis_host_list, redis_port, db_number);
}
if(!ldap_flag)
{
LOG(PURGER_LOG_ERR,"You must specify an ldap server.");
exit(EXIT_FAILURE);
}
else
ldap_init(ldap_server,ldap_port);
mailinfo.from = "*****@*****.**";
mailinfo.fromreal = "*****@*****.**";
mailinfo.subject = "[PURGER NOTIFICATION]";
mailinfo.defaultto = "*****@*****.**";
mailinfo.server = "mail.lanl.gov";
mailinfo.txt = "The following text file in the Turquiose network contains a list of \
your scratch files that have not been modified in the last 14+ days. \
Those files will be deleted in at least 6 days if not modified by then. \
This notification may not have up-to-the-minute information, but we \
will verify a file's actual age before purging it. For more information, \
please see our purge policy: http://hpc.lanl.gov/purge_policy. \
If you have questions or concerns, please contact ICN Consultants \
at 505-665-4444 option 3.";
time(&time_started);
if(warnusers_check_state(PURGER_global_rank,force_flag) < 0)
exit(1);
CIRCLE_cb_create(&add_objects);
CIRCLE_cb_process(&process_objects);
CIRCLE_begin();
CIRCLE_finalize();
time(&time_finished);
char starttime_str[256];
char endtime_str[256];
struct tm * localstart = localtime( &time_started );
struct tm * localend = localtime ( &time_finished );
strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
char getCmd[256];
sprintf(getCmd,"set warnusers_timestamp \"%s\"",endtime_str);
if(redis_blocking_command(getCmd,(void*)NULL,NIL)<0)
{
LOG(PURGER_LOG_ERR,"Unable to %s",getCmd);
}
sprintf(getCmd,"set warnusers-rank-%d 0", PURGER_global_rank);
if(redis_blocking_command(getCmd,(void*)NULL,NIL)<0)
{
LOG(PURGER_LOG_ERR,"Unable to %s",getCmd);
}
redis_finalize();
if(sharded_flag)
redis_shard_finalize();
if(PURGER_global_rank == 0)
{
LOG(PURGER_LOG_INFO, "Warnusers run started at: %s", starttime_str);
LOG(PURGER_LOG_INFO, "Warnusers run completed at: %s", endtime_str);
LOG(PURGER_LOG_INFO, "Warnusers total time (seconds) for this run: %f",difftime(time_finished,time_started));
}
exit(EXIT_SUCCESS);
}
int main(int argc, \
char** argv)
{
int c;
int option_index = 0;
MPI_Init(&argc, &argv);
/* Initialize our processing library and related callbacks. */
/* This is a bit of chicken-and-egg problem, because we'd like
* to have our rank to filter output messages below but we might
* also want to set different libcircle flags based on command line
* options -- for now just pass in the default flags */
CIRCLE_global_rank = CIRCLE_init(argc, argv, CIRCLE_DEFAULT_FLAGS);
CIRCLE_cb_create(&DCOPY_add_objects);
CIRCLE_cb_process(&DCOPY_process_objects);
DCOPY_debug_stream = stdout;
/* By default, don't perform a conditional copy. */
DCOPY_user_opts.conditional = false;
/* By default, don't skip the compare option. */
DCOPY_user_opts.skip_compare = false;
/* By default, show info log messages. */
CIRCLE_loglevel CIRCLE_debug = CIRCLE_LOG_INFO;
DCOPY_debug_level = DCOPY_LOG_DBG;
/* By default, don't unlink destination files if an open() fails. */
DCOPY_user_opts.force = false;
/* By default, don't bother to preserve all attributes. */
DCOPY_user_opts.preserve = false;
/* By default, don't attempt any type of recursion. */
DCOPY_user_opts.recursive = false;
DCOPY_user_opts.recursive_unspecified = false;
/* By default, assume the filesystem is reliable (exit on errors). */
DCOPY_user_opts.reliable_filesystem = true;
static struct option long_options[] = {
{"conditional" , no_argument , 0, 'c'},
{"skip-compare" , no_argument , 0, 'C'},
{"debug" , required_argument, 0, 'd'},
{"force" , no_argument , 0, 'f'},
{"help" , no_argument , 0, 'h'},
{"preserve" , no_argument , 0, 'p'},
{"recursive" , no_argument , 0, 'R'},
{"recursive-unspecified", no_argument , 0, 'r'},
{"unreliable-filesystem", no_argument , 0, 'U'},
{"version" , no_argument , 0, 'v'},
{0 , 0 , 0, 0 }
};
/* Parse options */
while((c = getopt_long(argc, argv, "cCd:fhpRrUv", \
long_options, &option_index)) != -1) {
switch(c) {
case 'c':
DCOPY_user_opts.conditional = true;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Performing a conditional copy.");
}
break;
case 'C':
DCOPY_user_opts.skip_compare = true;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Skipping the comparison stage " \
"(may result in corruption).");
}
break;
case 'd':
if(strncmp(optarg, "fatal", 5)) {
CIRCLE_debug = CIRCLE_LOG_FATAL;
DCOPY_debug_level = DCOPY_LOG_FATAL;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Debug level set to: fatal");
}
}
else if(strncmp(optarg, "err", 3)) {
CIRCLE_debug = CIRCLE_LOG_ERR;
DCOPY_debug_level = DCOPY_LOG_ERR;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Debug level set to: errors");
}
}
else if(strncmp(optarg, "warn", 4)) {
CIRCLE_debug = CIRCLE_LOG_WARN;
DCOPY_debug_level = DCOPY_LOG_WARN;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Debug level set to: warnings");
}
}
else if(strncmp(optarg, "info", 4)) {
CIRCLE_debug = CIRCLE_LOG_INFO;
DCOPY_debug_level = DCOPY_LOG_INFO;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Debug level set to: info");
}
}
else if(strncmp(optarg, "dbg", 4)) {
CIRCLE_debug = CIRCLE_LOG_DBG;
DCOPY_debug_level = DCOPY_LOG_DBG;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Debug level set to: debug");
}
}
else {
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Debug level `%s' not recognized. " \
"Defaulting to `info'.", optarg);
}
}
break;
case 'f':
DCOPY_user_opts.force = true;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Deleting destination on errors.");
}
break;
case 'h':
if(CIRCLE_global_rank == 0) {
DCOPY_print_usage(argv);
}
DCOPY_exit(EXIT_SUCCESS);
break;
case 'p':
DCOPY_user_opts.preserve = true;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Preserving file attributes.");
}
break;
case 'R':
DCOPY_user_opts.recursive = true;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Performing correct recursion.");
LOG(DCOPY_LOG_WARN, "Warning, only files and directories are implemented.");
}
break;
case 'r':
DCOPY_user_opts.recursive_unspecified = true;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Performing recursion. " \
"Ignoring special files.");
}
break;
case 'U':
DCOPY_user_opts.reliable_filesystem = false;
if(CIRCLE_global_rank == 0) {
LOG(DCOPY_LOG_INFO, "Unreliable filesystem specified. " \
"Retry mode enabled.");
}
break;
case 'v':
if(CIRCLE_global_rank == 0) {
DCOPY_print_version();
}
DCOPY_exit(EXIT_SUCCESS);
break;
case '?':
default:
if(CIRCLE_global_rank == 0) {
if(optopt == 'd') {
DCOPY_print_usage(argv);
fprintf(stderr, "Option -%c requires an argument.\n", \
optopt);
}
else if(isprint(optopt)) {
DCOPY_print_usage(argv);
fprintf(stderr, "Unknown option `-%c'.\n", optopt);
}
else {
DCOPY_print_usage(argv);
fprintf(stderr,
"Unknown option character `\\x%x'.\n",
optopt);
}
}
DCOPY_exit(EXIT_FAILURE);
break;
}
}
/** Parse the source and destination paths. */
DCOPY_parse_path_args(argv, optind, argc);
/* initialize linked list of stat objects */
DCOPY_list_head = NULL;
DCOPY_list_tail = NULL;
/* Initialize our jump table for core operations. */
DCOPY_jump_table[TREEWALK] = DCOPY_do_treewalk;
DCOPY_jump_table[COPY] = DCOPY_do_copy;
DCOPY_jump_table[CLEANUP] = DCOPY_do_cleanup;
DCOPY_jump_table[COMPARE] = DCOPY_do_compare;
/* Set the log level for the processing library. */
CIRCLE_enable_logging(CIRCLE_debug);
/* Grab a relative and actual start time for the epilogue. */
time(&(DCOPY_statistics.time_started));
DCOPY_statistics.wtime_started = CIRCLE_wtime();
/* Perform the actual file copy. */
CIRCLE_begin();
/* Determine the actual and relative end time for the epilogue. */
DCOPY_statistics.wtime_ended = CIRCLE_wtime();
time(&(DCOPY_statistics.time_ended));
/* Let the processing library cleanup. */
CIRCLE_finalize();
/* set timestamps if needed */
if (DCOPY_user_opts.preserve) {
DCOPY_set_timestamps();
}
/* free list of stat objects */
DCOPY_stat_elem_t* current = DCOPY_list_head;
while (current != NULL) {
DCOPY_stat_elem_t* next = current->next;
free(current->file);
free(current->sb);
free(current);
current = next;
}
/* Print the results to the user. */
DCOPY_epilogue();
DCOPY_exit(EXIT_SUCCESS);
}