예제 #1
0
파일: treewalk.c 프로젝트: jlafon/purger
/*
 * Main
 */
int
main (int argc, char **argv)
{
    /* Locals */
    char starttime_str[256];
    char endtime_str[256];
    char getCmd[256];

    treewalk_options_st opts;    
    treewalk_init_opts(&opts);

    /* Globals */
    treewalk_init_globals();

    /* Set up signal handler */
    treewalk_install_signal_handlers();

    /* Enable logging. */
    PURGER_debug_stream = stdout;
    PURGER_debug_level = PURGER_LOG_DBG;
    
    /* Init lib circle */
    int rank = CIRCLE_init(argc, argv);
    if(rank < 0)
        exit(1);
    CIRCLE_enable_logging(CIRCLE_LOG_ERR);
    PURGER_global_rank = rank;
    opts.rank = rank;

    /* Process command line options */    
    treewalk_process_options(argc,argv,&opts); 

    /* Init redis */
    if (!benchmarking_flag && redis_init(opts.redis_hostname,opts.redis_port,opts.db_number) < 0)
    {
        LOG(PURGER_LOG_FATAL, "Redis error: %s", REDIS->errstr);
        exit(EXIT_FAILURE);
    }
    
    /* Timing */
    time(&time_started);
    
    /* Ensure it's OK to run at this time */
    if(!benchmarking_flag && treewalk_check_state(opts.rank,opts.force_flag) < 0)
       exit(1);

    /* Read from restart files */
    if(!benchmarking_flag && opts.restart_flag)
        CIRCLE_read_restarts();

    /* Enable sharding */
    if(!benchmarking_flag && sharded_flag)
    {
        sharded_count = redis_shard_init(opts.redis_hostlist,opts.redis_port,opts.db_number);
        redis_command_ptr = &redis_shard_command;
    }

    /* Parallel section */
    CIRCLE_cb_create(&add_objects);
    CIRCLE_cb_process(&process_objects);
    CIRCLE_begin();
    /* End parallel section (well, kind of) */   

    /* Set state */ 
    sprintf(getCmd,"set treewalk-rank-%d 0", rank);
    if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT)<0)
    {
          fprintf(stderr,"Unable to %s",getCmd);
    }
     
    time(&time_finished);
    struct tm * localstart = localtime( &time_started );
    struct tm * localend = localtime ( &time_finished );
    strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
    strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
    sprintf(getCmd,"set treewalk_timestamp \"%s\"",endtime_str);
    if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT) < 0)
    {
        fprintf(stderr,"Unable to %s",getCmd);
    }
    LOG(PURGER_LOG_INFO,"Files: %d\tDirs: %d\tTotal: %d\n",file_count,dir_count,file_count+dir_count);
    if(!benchmarking_flag && sharded_flag)
        redis_shard_finalize();
    if(!benchmarking_flag)
        redis_finalize(); 
   
    if(rank == 0)
    {
        LOG(PURGER_LOG_INFO, "treewalk run started at: %s", starttime_str);
        LOG(PURGER_LOG_INFO, "treewalk run completed at: %s", endtime_str);
        LOG(PURGER_LOG_INFO, "treewalk total time (seconds) for this run: %f",difftime(time_finished,time_started));
    }
        LOG(PURGER_LOG_INFO, "\nTotal time in process_objects: %lf\n\
                   \tRedis commands: %lf %lf%%\n\
                   \tStating:  %lf %lf%%\n\
                   \tReaddir: %lf %lf%%\n\
                   \tHashing: %lf %lf%%\n",
                   process_objects_total[1],redis_time[1],redis_time[1]/process_objects_total[1]*100.0,stat_time[1],stat_time[1]/process_objects_total[1]*100.0,readdir_time[1],readdir_time[1]/process_objects_total[1]*100.0
                   ,hash_time[1],hash_time[1]/process_objects_total[1]*100.0);
    CIRCLE_finalize();
    _exit(EXIT_SUCCESS);
}
예제 #2
0
int
main (int argc, char **argv)
{
    int index;
    int c;

    char *redis_hostname;
    char *redis_hostlist;
    int redis_port;

    int time_flag = 0;
    int dir_flag = 0;
    int force_flag = 0;
    int restart_flag = 0;
    int redis_hostname_flag = 0;
    benchmarking_flag = 0;
    sharded_flag = 0;
    int redis_port_flag = 0;

    process_objects_total[2] = 0;
    redis_time[2] = 0;
    stat_time[2] = 0;
    readdir_time[2] = 0;
    redis_command_ptr = &redis_command;

    
    /* Enable logging. */
    PURGER_debug_stream = stdout;
    PURGER_debug_level = PURGER_LOG_DBG;
    int rank = CIRCLE_init(argc, argv);
    PURGER_global_rank = rank;
    opterr = 0;
    while((c = getopt(argc, argv, "d:h:p:ft:l:rs:b")) != -1)
    {
        switch(c)
        {
            case 'b':
		benchmarking_flag = 1;
		break;
            case 'd':
                TOP_DIR = realpath(optarg, NULL);
                if(rank == 0) LOG(PURGER_LOG_INFO,"Using %s as a root path.",TOP_DIR);
                dir_flag = 1;
                break;
        
            case 'l':
                PURGER_debug_level = atoi(optarg);
                break;

            case 'h':
                redis_hostname = optarg;
                redis_hostname_flag = 1;
                break;

            case 'p':
                redis_port = atoi(optarg);
                redis_port_flag = 1;
                break;
            
            case 'r':
                if(rank == 0) LOG(PURGER_LOG_WARN,"You have specified to use restart files.");
                restart_flag = 1;
                break;

            case 't':
                time_flag = 1;
                expire_threshold = (float)SECONDS_PER_DAY * atof(optarg);
                if(rank == 0) LOG(PURGER_LOG_WARN,"Changed file expiration time to %.2f days, or %.2f seconds.",expire_threshold/(60.0*60.0*24),expire_threshold);
                break;
            case 's':
                sharded_flag = 1;
                redis_hostlist = optarg;
                break;

            case 'f':
                force_flag = 1;
                if(rank == 0) LOG(PURGER_LOG_WARN,"Warning: You have chosen to force treewalk.");
                break;
            
            case '?':
                if (optopt == 'd' || optopt == 'h' || optopt == 'p' || optopt == 't' || optopt == 'l' || optopt == 's')
                {
                    print_usage(argv);
                    fprintf(stderr, "Option -%c requires an argument.\n", optopt);
                }
                else if (isprint (optopt))
                {
                    print_usage(argv);
                    fprintf(stderr, "Unknown option `-%c'.\n", optopt);
                }
                else
                {
                    print_usage(argv);
                    fprintf(stderr,
                        "Unknown option character `\\x%x'.\n",
                        optopt);
                }
                exit(EXIT_FAILURE);
            default:
                abort();
        }
    }
    if(restart_flag && dir_flag && !benchmarking_flag)
    {
        if(rank == 0) LOG(PURGER_LOG_WARN, "You have told treewalk to use both checkpoint files and a directory.  You cannot combine these options.\n"
                                    "If you use a directory, treewalk will start from scratch.  If you use a checkpoint file, it will start from\n"
                                    "from the data in the checkpoint files.\n");
        exit(EXIT_FAILURE);
    }
    if(time_flag == 0 && !benchmarking_flag)
    {
        if(rank == 0) LOG(PURGER_LOG_WARN, "A file timeout value was not specified.  Files older than %.2f seconds (%.2f days) will be expired.",expire_threshold,expire_threshold/(60.0*60.0*24.0));
    }

    if(dir_flag == 0 && !restart_flag && !benchmarking_flag)
    {
         print_usage(argv);
         if(rank == 0) LOG(PURGER_LOG_FATAL, "You must specify a starting directory");
         exit(EXIT_FAILURE);
    }

    if(redis_hostname_flag == 0 && !benchmarking_flag)
    {
        if(rank == 0) LOG(PURGER_LOG_WARN, "A hostname for redis was not specified, defaulting to localhost.");
        redis_hostname = "localhost";
    }

    if(redis_port_flag == 0 && !benchmarking_flag)
    {
        if(rank == 0) LOG(PURGER_LOG_WARN, "A port number for redis was not specified, defaulting to 6379.");
        redis_port = 6379;
    }

    for (index = optind; index < argc; index++)
        LOG(PURGER_LOG_WARN, "Non-option argument %s", argv[index]);
    if (!benchmarking_flag && redis_init(redis_hostname,redis_port) < 0)
    {
        LOG(PURGER_LOG_FATAL, "Redis error: %s", REDIS->errstr);
        exit(EXIT_FAILURE);
    }
    

   time(&time_started);
   if(!benchmarking_flag && treewalk_check_state(rank,force_flag) < 0)
       exit(1);
    if(!benchmarking_flag && restart_flag)
        CIRCLE_read_restarts();
    if(!benchmarking_flag && sharded_flag)
    {
        sharded_count = redis_shard_init(redis_hostlist,redis_port);
        redis_command_ptr = &redis_shard_command;
    }
    CIRCLE_cb_create(&add_objects);
    CIRCLE_cb_process(&process_objects);
    CIRCLE_begin();
    CIRCLE_finalize();
    
    char getCmd[256];
    sprintf(getCmd,"set treewalk-rank-%d 0", rank);
    if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT)<0)
    {
        fprintf(stderr,"Unable to %s",getCmd);
        exit(1);
    }

    time(&time_finished);
    char starttime_str[256];
    char endtime_str[256];
    struct tm * localstart = localtime( &time_started );
    struct tm * localend = localtime ( &time_finished );
    strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
    strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
    sprintf(getCmd,"set treewalk_timestamp \"%s\"",endtime_str);
    if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT) < 0)
    {
        fprintf(stderr,"Unable to %s",getCmd);
    }
    
    if(rank == 0)
    {
        LOG(PURGER_LOG_INFO, "treewalk run started at: %s", starttime_str);
        LOG(PURGER_LOG_INFO, "treewalk run completed at: %s", endtime_str);
        LOG(PURGER_LOG_INFO, "treewalk total time (seconds) for this run: %f",difftime(time_finished,time_started));
        LOG(PURGER_LOG_INFO, "\nTotal time in process_objects: %lf\n\
                   \tRedis commands: %lf %lf\n\
                   \tStating:  %lf %lf\n\
                   \tReaddir: %lf %lf\n\
                   \tHashing: %lf %lf\n",
                   process_objects_total[1],redis_time[1],redis_time[1]/process_objects_total[1]*100.0,stat_time[1],stat_time[1]/process_objects_total[1]*100.0,readdir_time[1],readdir_time[1]/process_objects_total[1]*100.0
                   ,hash_time[1],hash_time[1]/process_objects_total[1]*100.0);
    }