int
main (int argc, char **argv)
{
int index;
int c;
char *redis_hostname;
char *redis_hostlist;
int redis_port;
int time_flag = 0;
int dir_flag = 0;
int force_flag = 0;
int restart_flag = 0;
int redis_hostname_flag = 0;
benchmarking_flag = 0;
sharded_flag = 0;
int redis_port_flag = 0;
process_objects_total[2] = 0;
redis_time[2] = 0;
stat_time[2] = 0;
readdir_time[2] = 0;
redis_command_ptr = &redis_command;
/* Enable logging. */
PURGER_debug_stream = stdout;
PURGER_debug_level = PURGER_LOG_DBG;
int rank = CIRCLE_init(argc, argv);
PURGER_global_rank = rank;
opterr = 0;
while((c = getopt(argc, argv, "d:h:p:ft:l:rs:b")) != -1)
{
switch(c)
{
case 'b':
benchmarking_flag = 1;
break;
case 'd':
TOP_DIR = realpath(optarg, NULL);
if(rank == 0) LOG(PURGER_LOG_INFO,"Using %s as a root path.",TOP_DIR);
dir_flag = 1;
break;
case 'l':
PURGER_debug_level = atoi(optarg);
break;
case 'h':
redis_hostname = optarg;
redis_hostname_flag = 1;
break;
case 'p':
redis_port = atoi(optarg);
redis_port_flag = 1;
break;
case 'r':
if(rank == 0) LOG(PURGER_LOG_WARN,"You have specified to use restart files.");
restart_flag = 1;
break;
case 't':
time_flag = 1;
expire_threshold = (float)SECONDS_PER_DAY * atof(optarg);
if(rank == 0) LOG(PURGER_LOG_WARN,"Changed file expiration time to %.2f days, or %.2f seconds.",expire_threshold/(60.0*60.0*24),expire_threshold);
break;
case 's':
sharded_flag = 1;
redis_hostlist = optarg;
break;
case 'f':
force_flag = 1;
if(rank == 0) LOG(PURGER_LOG_WARN,"Warning: You have chosen to force treewalk.");
break;
case '?':
if (optopt == 'd' || optopt == 'h' || optopt == 'p' || optopt == 't' || optopt == 'l' || optopt == 's')
{
print_usage(argv);
fprintf(stderr, "Option -%c requires an argument.\n", optopt);
}
else if (isprint (optopt))
{
print_usage(argv);
fprintf(stderr, "Unknown option `-%c'.\n", optopt);
}
else
{
print_usage(argv);
fprintf(stderr,
"Unknown option character `\\x%x'.\n",
optopt);
}
exit(EXIT_FAILURE);
default:
abort();
}
}
if(restart_flag && dir_flag && !benchmarking_flag)
{
if(rank == 0) LOG(PURGER_LOG_WARN, "You have told treewalk to use both checkpoint files and a directory. You cannot combine these options.\n"
"If you use a directory, treewalk will start from scratch. If you use a checkpoint file, it will start from\n"
"from the data in the checkpoint files.\n");
exit(EXIT_FAILURE);
}
if(time_flag == 0 && !benchmarking_flag)
{
if(rank == 0) LOG(PURGER_LOG_WARN, "A file timeout value was not specified. Files older than %.2f seconds (%.2f days) will be expired.",expire_threshold,expire_threshold/(60.0*60.0*24.0));
}
if(dir_flag == 0 && !restart_flag && !benchmarking_flag)
{
print_usage(argv);
if(rank == 0) LOG(PURGER_LOG_FATAL, "You must specify a starting directory");
exit(EXIT_FAILURE);
}
if(redis_hostname_flag == 0 && !benchmarking_flag)
{
if(rank == 0) LOG(PURGER_LOG_WARN, "A hostname for redis was not specified, defaulting to localhost.");
redis_hostname = "localhost";
}
if(redis_port_flag == 0 && !benchmarking_flag)
{
if(rank == 0) LOG(PURGER_LOG_WARN, "A port number for redis was not specified, defaulting to 6379.");
redis_port = 6379;
}
for (index = optind; index < argc; index++)
LOG(PURGER_LOG_WARN, "Non-option argument %s", argv[index]);
if (!benchmarking_flag && redis_init(redis_hostname,redis_port) < 0)
{
LOG(PURGER_LOG_FATAL, "Redis error: %s", REDIS->errstr);
exit(EXIT_FAILURE);
}
time(&time_started);
if(!benchmarking_flag && treewalk_check_state(rank,force_flag) < 0)
exit(1);
if(!benchmarking_flag && restart_flag)
CIRCLE_read_restarts();
if(!benchmarking_flag && sharded_flag)
{
sharded_count = redis_shard_init(redis_hostlist,redis_port);
redis_command_ptr = &redis_shard_command;
}
CIRCLE_cb_create(&add_objects);
CIRCLE_cb_process(&process_objects);
CIRCLE_begin();
CIRCLE_finalize();
char getCmd[256];
sprintf(getCmd,"set treewalk-rank-%d 0", rank);
if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT)<0)
{
fprintf(stderr,"Unable to %s",getCmd);
exit(1);
}
time(&time_finished);
char starttime_str[256];
char endtime_str[256];
struct tm * localstart = localtime( &time_started );
struct tm * localend = localtime ( &time_finished );
strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
sprintf(getCmd,"set treewalk_timestamp \"%s\"",endtime_str);
if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT) < 0)
{
fprintf(stderr,"Unable to %s",getCmd);
}
if(rank == 0)
{
LOG(PURGER_LOG_INFO, "treewalk run started at: %s", starttime_str);
LOG(PURGER_LOG_INFO, "treewalk run completed at: %s", endtime_str);
LOG(PURGER_LOG_INFO, "treewalk total time (seconds) for this run: %f",difftime(time_finished,time_started));
LOG(PURGER_LOG_INFO, "\nTotal time in process_objects: %lf\n\
\tRedis commands: %lf %lf\n\
\tStating: %lf %lf\n\
\tReaddir: %lf %lf\n\
\tHashing: %lf %lf\n",
process_objects_total[1],redis_time[1],redis_time[1]/process_objects_total[1]*100.0,stat_time[1],stat_time[1]/process_objects_total[1]*100.0,readdir_time[1],readdir_time[1]/process_objects_total[1]*100.0
,hash_time[1],hash_time[1]/process_objects_total[1]*100.0);
}
/*
* Main
*/
int
main (int argc, char **argv)
{
/* Locals */
char starttime_str[256];
char endtime_str[256];
char getCmd[256];
treewalk_options_st opts;
treewalk_init_opts(&opts);
/* Globals */
treewalk_init_globals();
/* Set up signal handler */
treewalk_install_signal_handlers();
/* Enable logging. */
PURGER_debug_stream = stdout;
PURGER_debug_level = PURGER_LOG_DBG;
/* Init lib circle */
int rank = CIRCLE_init(argc, argv);
if(rank < 0)
exit(1);
CIRCLE_enable_logging(CIRCLE_LOG_ERR);
PURGER_global_rank = rank;
opts.rank = rank;
/* Process command line options */
treewalk_process_options(argc,argv,&opts);
/* Init redis */
if (!benchmarking_flag && redis_init(opts.redis_hostname,opts.redis_port,opts.db_number) < 0)
{
LOG(PURGER_LOG_FATAL, "Redis error: %s", REDIS->errstr);
exit(EXIT_FAILURE);
}
/* Timing */
time(&time_started);
/* Ensure it's OK to run at this time */
if(!benchmarking_flag && treewalk_check_state(opts.rank,opts.force_flag) < 0)
exit(1);
/* Read from restart files */
if(!benchmarking_flag && opts.restart_flag)
CIRCLE_read_restarts();
/* Enable sharding */
if(!benchmarking_flag && sharded_flag)
{
sharded_count = redis_shard_init(opts.redis_hostlist,opts.redis_port,opts.db_number);
redis_command_ptr = &redis_shard_command;
}
/* Parallel section */
CIRCLE_cb_create(&add_objects);
CIRCLE_cb_process(&process_objects);
CIRCLE_begin();
/* End parallel section (well, kind of) */
/* Set state */
sprintf(getCmd,"set treewalk-rank-%d 0", rank);
if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT)<0)
{
fprintf(stderr,"Unable to %s",getCmd);
}
time(&time_finished);
struct tm * localstart = localtime( &time_started );
struct tm * localend = localtime ( &time_finished );
strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
sprintf(getCmd,"set treewalk_timestamp \"%s\"",endtime_str);
if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT) < 0)
{
fprintf(stderr,"Unable to %s",getCmd);
}
LOG(PURGER_LOG_INFO,"Files: %d\tDirs: %d\tTotal: %d\n",file_count,dir_count,file_count+dir_count);
if(!benchmarking_flag && sharded_flag)
redis_shard_finalize();
if(!benchmarking_flag)
redis_finalize();
if(rank == 0)
{
LOG(PURGER_LOG_INFO, "treewalk run started at: %s", starttime_str);
LOG(PURGER_LOG_INFO, "treewalk run completed at: %s", endtime_str);
LOG(PURGER_LOG_INFO, "treewalk total time (seconds) for this run: %f",difftime(time_finished,time_started));
}
LOG(PURGER_LOG_INFO, "\nTotal time in process_objects: %lf\n\
\tRedis commands: %lf %lf%%\n\
\tStating: %lf %lf%%\n\
\tReaddir: %lf %lf%%\n\
\tHashing: %lf %lf%%\n",
process_objects_total[1],redis_time[1],redis_time[1]/process_objects_total[1]*100.0,stat_time[1],stat_time[1]/process_objects_total[1]*100.0,readdir_time[1],readdir_time[1]/process_objects_total[1]*100.0
,hash_time[1],hash_time[1]/process_objects_total[1]*100.0);
CIRCLE_finalize();
_exit(EXIT_SUCCESS);
}
int
main (int argc, char **argv)
{
int index;
int c;
char *redis_hostname;
int redis_port;
int redis_hostname_flag = 0;
int redis_port_flag = 0;
int sharded_flag = 0;
int ldap_flag = 0;
int force_flag = 0;
int db_number = 0;
sharded_count = 0;
char * redis_host_list;
redis_command_ptr = &non_sharded_redis_command;
PURGER_debug_stream = stdout;
PURGER_debug_level = PURGER_LOG_DBG;
int PURGER_global_rank = CIRCLE_init(argc, argv);
if(PURGER_global_rank < 0)
exit(1);
opterr = 0;
CIRCLE_enable_logging(CIRCLE_LOG_ERR);
while((c = getopt(argc, argv, "h:p:l:fi:s:a:o:")) != -1)
{
switch(c)
{
case 'o':
ldap_port = atoi(optarg);
break;
case 'a':
ldap_server = optarg;
ldap_flag = 1;
break;
case 'h':
redis_hostname = optarg;
redis_hostname_flag = 1;
break;
case 'i':
db_number = atoi(optarg);
break;
case 'p':
redis_port = atoi(optarg);
redis_port_flag = 1;
break;
case 'f':
force_flag = 1;
if(PURGER_global_rank == 0) LOG(PURGER_LOG_WARN,"Warning: You have chosen to force warnusers.");
break;
case 'l':
PURGER_debug_level = atoi(optarg);
break;
case 's':
redis_host_list = optarg;
sharded_flag = 1;
redis_command_ptr = &redis_blocking_shard_command;
break;
case '?':
if (PURGER_global_rank == 0 && (optopt == 'h' || optopt == 'p' || optopt == 'l'))
{
print_usage(argv);
fprintf(stderr, "Option -%c requires an argument.\n", optopt);
exit(EXIT_FAILURE);
}
else if (isprint (optopt))
{
print_usage(argv);
fprintf(stderr, "Unknown option `-%c'.\n", optopt);
exit(EXIT_FAILURE);
}
else
{
print_usage(argv);
fprintf(stderr,
"Unknown option character `\\x%x'.\n",
optopt);
exit(EXIT_FAILURE);
}
default:
abort();
}
}
if(redis_hostname_flag == 0)
{
if(PURGER_global_rank == 0) LOG(PURGER_LOG_WARN, "A hostname for redis was not specified, defaulting to localhost.");
redis_hostname = "localhost";
}
if(redis_port_flag == 0)
{
if(PURGER_global_rank == 0) LOG(PURGER_LOG_WARN, "A port number for redis was not specified, defaulting to 6379.");
redis_port = 6379;
}
for (index = optind; index < argc; index++)
LOG(PURGER_LOG_WARN, "Non-option argument %s", argv[index]);
if(redis_init(redis_hostname, redis_port, db_number) < 0)
{
LOG(PURGER_LOG_ERR,"Unable to initialize redis.");
exit(0);
}
if(sharded_flag)
{
LOG(PURGER_LOG_INFO,"Sharding enabled.");
sharded_count = redis_shard_init(redis_host_list, redis_port, db_number);
}
if(!ldap_flag)
{
LOG(PURGER_LOG_ERR,"You must specify an ldap server.");
exit(EXIT_FAILURE);
}
else
ldap_init(ldap_server,ldap_port);
mailinfo.from = "*****@*****.**";
mailinfo.fromreal = "*****@*****.**";
mailinfo.subject = "[PURGER NOTIFICATION]";
mailinfo.defaultto = "*****@*****.**";
mailinfo.server = "mail.lanl.gov";
mailinfo.txt = "The following text file in the Turquiose network contains a list of \
your scratch files that have not been modified in the last 14+ days. \
Those files will be deleted in at least 6 days if not modified by then. \
This notification may not have up-to-the-minute information, but we \
will verify a file's actual age before purging it. For more information, \
please see our purge policy: http://hpc.lanl.gov/purge_policy. \
If you have questions or concerns, please contact ICN Consultants \
at 505-665-4444 option 3.";
time(&time_started);
if(warnusers_check_state(PURGER_global_rank,force_flag) < 0)
exit(1);
CIRCLE_cb_create(&add_objects);
CIRCLE_cb_process(&process_objects);
CIRCLE_begin();
CIRCLE_finalize();
time(&time_finished);
char starttime_str[256];
char endtime_str[256];
struct tm * localstart = localtime( &time_started );
struct tm * localend = localtime ( &time_finished );
strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
char getCmd[256];
sprintf(getCmd,"set warnusers_timestamp \"%s\"",endtime_str);
if(redis_blocking_command(getCmd,(void*)NULL,NIL)<0)
{
LOG(PURGER_LOG_ERR,"Unable to %s",getCmd);
}
sprintf(getCmd,"set warnusers-rank-%d 0", PURGER_global_rank);
if(redis_blocking_command(getCmd,(void*)NULL,NIL)<0)
{
LOG(PURGER_LOG_ERR,"Unable to %s",getCmd);
}
redis_finalize();
if(sharded_flag)
redis_shard_finalize();
if(PURGER_global_rank == 0)
{
LOG(PURGER_LOG_INFO, "Warnusers run started at: %s", starttime_str);
LOG(PURGER_LOG_INFO, "Warnusers run completed at: %s", endtime_str);
LOG(PURGER_LOG_INFO, "Warnusers total time (seconds) for this run: %f",difftime(time_finished,time_started));
}
exit(EXIT_SUCCESS);
}