コード例 #1
0
/* insert all items to be removed into libcircle for
 * dynamic load balancing */
static void remove_libcircle(mfu_flist list, uint64_t* rmcount)
{
    /* set globals for libcircle callbacks */
    circle_list  = list;
    circle_count = 0;

    /* initialize libcircle */
    CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL);

    /* set libcircle verbosity level */
    enum CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
    if (mfu_debug_level >= MFU_LOG_VERBOSE) {
        //        loglevel = CIRCLE_LOG_INFO;
    }
    CIRCLE_enable_logging(loglevel);

    /* register callbacks */
    CIRCLE_cb_create(&remove_create);
    CIRCLE_cb_process(&remove_process);

    /* run the libcircle job */
    CIRCLE_begin();
    CIRCLE_finalize();

    /* record number of items we deleted */
    *rmcount = circle_count;

    return;
}
コード例 #2
0
ファイル: dparallel.c プロジェクト: KnightKu/fileutils
int main(void)
{
    DPARALLEL_debug_level = DPARALLEL_LOG_DBG;
    DPARALLEL_debug_stream = stderr;

    CIRCLE_init(0, NULL, CIRCLE_DEFAULT_FLAGS);
    CIRCLE_cb_process(&DPARALLEL_process);

    CIRCLE_begin();
    CIRCLE_finalize();
}
コード例 #3
0
ファイル: treewalk.c プロジェクト: jlafon/purger
/*
 * Main
 */
int
main (int argc, char **argv)
{
    /* Locals */
    char starttime_str[256];
    char endtime_str[256];
    char getCmd[256];

    treewalk_options_st opts;    
    treewalk_init_opts(&opts);

    /* Globals */
    treewalk_init_globals();

    /* Set up signal handler */
    treewalk_install_signal_handlers();

    /* Enable logging. */
    PURGER_debug_stream = stdout;
    PURGER_debug_level = PURGER_LOG_DBG;
    
    /* Init lib circle */
    int rank = CIRCLE_init(argc, argv);
    if(rank < 0)
        exit(1);
    CIRCLE_enable_logging(CIRCLE_LOG_ERR);
    PURGER_global_rank = rank;
    opts.rank = rank;

    /* Process command line options */    
    treewalk_process_options(argc,argv,&opts); 

    /* Init redis */
    if (!benchmarking_flag && redis_init(opts.redis_hostname,opts.redis_port,opts.db_number) < 0)
    {
        LOG(PURGER_LOG_FATAL, "Redis error: %s", REDIS->errstr);
        exit(EXIT_FAILURE);
    }
    
    /* Timing */
    time(&time_started);
    
    /* Ensure it's OK to run at this time */
    if(!benchmarking_flag && treewalk_check_state(opts.rank,opts.force_flag) < 0)
       exit(1);

    /* Read from restart files */
    if(!benchmarking_flag && opts.restart_flag)
        CIRCLE_read_restarts();

    /* Enable sharding */
    if(!benchmarking_flag && sharded_flag)
    {
        sharded_count = redis_shard_init(opts.redis_hostlist,opts.redis_port,opts.db_number);
        redis_command_ptr = &redis_shard_command;
    }

    /* Parallel section */
    CIRCLE_cb_create(&add_objects);
    CIRCLE_cb_process(&process_objects);
    CIRCLE_begin();
    /* End parallel section (well, kind of) */   

    /* Set state */ 
    sprintf(getCmd,"set treewalk-rank-%d 0", rank);
    if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT)<0)
    {
          fprintf(stderr,"Unable to %s",getCmd);
    }
     
    time(&time_finished);
    struct tm * localstart = localtime( &time_started );
    struct tm * localend = localtime ( &time_finished );
    strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
    strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
    sprintf(getCmd,"set treewalk_timestamp \"%s\"",endtime_str);
    if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT) < 0)
    {
        fprintf(stderr,"Unable to %s",getCmd);
    }
    LOG(PURGER_LOG_INFO,"Files: %d\tDirs: %d\tTotal: %d\n",file_count,dir_count,file_count+dir_count);
    if(!benchmarking_flag && sharded_flag)
        redis_shard_finalize();
    if(!benchmarking_flag)
        redis_finalize(); 
   
    if(rank == 0)
    {
        LOG(PURGER_LOG_INFO, "treewalk run started at: %s", starttime_str);
        LOG(PURGER_LOG_INFO, "treewalk run completed at: %s", endtime_str);
        LOG(PURGER_LOG_INFO, "treewalk total time (seconds) for this run: %f",difftime(time_finished,time_started));
    }
        LOG(PURGER_LOG_INFO, "\nTotal time in process_objects: %lf\n\
                   \tRedis commands: %lf %lf%%\n\
                   \tStating:  %lf %lf%%\n\
                   \tReaddir: %lf %lf%%\n\
                   \tHashing: %lf %lf%%\n",
                   process_objects_total[1],redis_time[1],redis_time[1]/process_objects_total[1]*100.0,stat_time[1],stat_time[1]/process_objects_total[1]*100.0,readdir_time[1],readdir_time[1]/process_objects_total[1]*100.0
                   ,hash_time[1],hash_time[1]/process_objects_total[1]*100.0);
    CIRCLE_finalize();
    _exit(EXIT_SUCCESS);
}
コード例 #4
0
ファイル: mfu_flist_walk.c プロジェクト: hpc/fileutils
/* Set up and execute directory walk */
void mfu_flist_walk_paths(uint64_t num_paths, const char** paths,
                          mfu_walk_opts_t* walk_opts, mfu_flist bflist)
{
    /* report walk count, time, and rate */
    double start_walk = MPI_Wtime();

    /* if dir_permission is set to 1 then set global variable */
    SET_DIR_PERMS = 0;
    if (walk_opts->dir_perms) {
        SET_DIR_PERMS = 1;
    }

    /* if remove is set to 1 then set global variable */
    REMOVE_FILES  = 0;
    if (walk_opts->remove) {
        REMOVE_FILES = 1;
    }

    /* convert handle to flist_t */
    flist_t* flist = (flist_t*) bflist;

    /* get our rank and number of ranks in job */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* print message to user that we're starting */
    if (mfu_debug_level >= MFU_LOG_VERBOSE && mfu_rank == 0) {
        uint64_t i;
        for (i = 0; i < num_paths; i++) {
            MFU_LOG(MFU_LOG_INFO, "Walking %s", paths[i]);
        }
    }

    /* initialize libcircle */
    CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL);

    /* set libcircle verbosity level */
    enum CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
    CIRCLE_enable_logging(loglevel);

    /* TODO: check that paths is not NULL */
    /* TODO: check that each path is within limits */

    /* set some global variables to do the file walk */
    CURRENT_NUM_DIRS = num_paths;
    CURRENT_DIRS     = paths;
    CURRENT_LIST     = flist;

    /* we lookup users and groups first in case we can use
     * them to filter the walk */
    flist->detail = 0;
    if (walk_opts->use_stat) {
        flist->detail = 1;
        if (flist->have_users == 0) {
            mfu_flist_usrgrp_get_users(flist);
        }
        if (flist->have_groups == 0) {
            mfu_flist_usrgrp_get_groups(flist);
        }
    }

    /* register callbacks */
    if (walk_opts->use_stat) {
        /* walk directories by calling stat on every item */
        CIRCLE_cb_create(&walk_stat_create);
        CIRCLE_cb_process(&walk_stat_process);
        //        CIRCLE_cb_create(&walk_lustrestat_create);
        //        CIRCLE_cb_process(&walk_lustrestat_process);
    }
    else {
        /* walk directories using file types in readdir */
        CIRCLE_cb_create(&walk_readdir_create);
        CIRCLE_cb_process(&walk_readdir_process);
        //        CIRCLE_cb_create(&walk_getdents_create);
        //        CIRCLE_cb_process(&walk_getdents_process);
    }

    /* prepare callbacks and initialize variables for reductions */
    reduce_items = 0;
    CIRCLE_cb_reduce_init(&reduce_init);
    CIRCLE_cb_reduce_op(&reduce_exec);
    CIRCLE_cb_reduce_fini(&reduce_fini);

    /* run the libcircle job */
    CIRCLE_begin();
    CIRCLE_finalize();

    /* compute global summary */
    mfu_flist_summarize(bflist);

    double end_walk = MPI_Wtime();

    /* report walk count, time, and rate */
    if (mfu_debug_level >= MFU_LOG_VERBOSE && mfu_rank == 0) {
        uint64_t all_count = mfu_flist_global_size(bflist);
        double time_diff = end_walk - start_walk;
        double rate = 0.0;
        if (time_diff > 0.0) {
            rate = ((double)all_count) / time_diff;
        }
        MFU_LOG(MFU_LOG_INFO, "Walked %lu items in %f seconds (%f files/sec)",
               all_count, time_diff, rate
              );
    }

    /* hold procs here until summary is printed */
    MPI_Barrier(MPI_COMM_WORLD);

    return;
}
コード例 #5
0
int
main (int argc, char **argv)
{
    int index;
    int c;

    char *redis_hostname;
    char *redis_hostlist;
    int redis_port;

    int time_flag = 0;
    int dir_flag = 0;
    int force_flag = 0;
    int restart_flag = 0;
    int redis_hostname_flag = 0;
    benchmarking_flag = 0;
    sharded_flag = 0;
    int redis_port_flag = 0;

    process_objects_total[2] = 0;
    redis_time[2] = 0;
    stat_time[2] = 0;
    readdir_time[2] = 0;
    redis_command_ptr = &redis_command;

    
    /* Enable logging. */
    PURGER_debug_stream = stdout;
    PURGER_debug_level = PURGER_LOG_DBG;
    int rank = CIRCLE_init(argc, argv);
    PURGER_global_rank = rank;
    opterr = 0;
    while((c = getopt(argc, argv, "d:h:p:ft:l:rs:b")) != -1)
    {
        switch(c)
        {
            case 'b':
		benchmarking_flag = 1;
		break;
            case 'd':
                TOP_DIR = realpath(optarg, NULL);
                if(rank == 0) LOG(PURGER_LOG_INFO,"Using %s as a root path.",TOP_DIR);
                dir_flag = 1;
                break;
        
            case 'l':
                PURGER_debug_level = atoi(optarg);
                break;

            case 'h':
                redis_hostname = optarg;
                redis_hostname_flag = 1;
                break;

            case 'p':
                redis_port = atoi(optarg);
                redis_port_flag = 1;
                break;
            
            case 'r':
                if(rank == 0) LOG(PURGER_LOG_WARN,"You have specified to use restart files.");
                restart_flag = 1;
                break;

            case 't':
                time_flag = 1;
                expire_threshold = (float)SECONDS_PER_DAY * atof(optarg);
                if(rank == 0) LOG(PURGER_LOG_WARN,"Changed file expiration time to %.2f days, or %.2f seconds.",expire_threshold/(60.0*60.0*24),expire_threshold);
                break;
            case 's':
                sharded_flag = 1;
                redis_hostlist = optarg;
                break;

            case 'f':
                force_flag = 1;
                if(rank == 0) LOG(PURGER_LOG_WARN,"Warning: You have chosen to force treewalk.");
                break;
            
            case '?':
                if (optopt == 'd' || optopt == 'h' || optopt == 'p' || optopt == 't' || optopt == 'l' || optopt == 's')
                {
                    print_usage(argv);
                    fprintf(stderr, "Option -%c requires an argument.\n", optopt);
                }
                else if (isprint (optopt))
                {
                    print_usage(argv);
                    fprintf(stderr, "Unknown option `-%c'.\n", optopt);
                }
                else
                {
                    print_usage(argv);
                    fprintf(stderr,
                        "Unknown option character `\\x%x'.\n",
                        optopt);
                }
                exit(EXIT_FAILURE);
            default:
                abort();
        }
    }
    if(restart_flag && dir_flag && !benchmarking_flag)
    {
        if(rank == 0) LOG(PURGER_LOG_WARN, "You have told treewalk to use both checkpoint files and a directory.  You cannot combine these options.\n"
                                    "If you use a directory, treewalk will start from scratch.  If you use a checkpoint file, it will start from\n"
                                    "from the data in the checkpoint files.\n");
        exit(EXIT_FAILURE);
    }
    if(time_flag == 0 && !benchmarking_flag)
    {
        if(rank == 0) LOG(PURGER_LOG_WARN, "A file timeout value was not specified.  Files older than %.2f seconds (%.2f days) will be expired.",expire_threshold,expire_threshold/(60.0*60.0*24.0));
    }

    if(dir_flag == 0 && !restart_flag && !benchmarking_flag)
    {
         print_usage(argv);
         if(rank == 0) LOG(PURGER_LOG_FATAL, "You must specify a starting directory");
         exit(EXIT_FAILURE);
    }

    if(redis_hostname_flag == 0 && !benchmarking_flag)
    {
        if(rank == 0) LOG(PURGER_LOG_WARN, "A hostname for redis was not specified, defaulting to localhost.");
        redis_hostname = "localhost";
    }

    if(redis_port_flag == 0 && !benchmarking_flag)
    {
        if(rank == 0) LOG(PURGER_LOG_WARN, "A port number for redis was not specified, defaulting to 6379.");
        redis_port = 6379;
    }

    for (index = optind; index < argc; index++)
        LOG(PURGER_LOG_WARN, "Non-option argument %s", argv[index]);
    if (!benchmarking_flag && redis_init(redis_hostname,redis_port) < 0)
    {
        LOG(PURGER_LOG_FATAL, "Redis error: %s", REDIS->errstr);
        exit(EXIT_FAILURE);
    }
    

   time(&time_started);
   if(!benchmarking_flag && treewalk_check_state(rank,force_flag) < 0)
       exit(1);
    if(!benchmarking_flag && restart_flag)
        CIRCLE_read_restarts();
    if(!benchmarking_flag && sharded_flag)
    {
        sharded_count = redis_shard_init(redis_hostlist,redis_port);
        redis_command_ptr = &redis_shard_command;
    }
    CIRCLE_cb_create(&add_objects);
    CIRCLE_cb_process(&process_objects);
    CIRCLE_begin();
    CIRCLE_finalize();
    
    char getCmd[256];
    sprintf(getCmd,"set treewalk-rank-%d 0", rank);
    if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT)<0)
    {
        fprintf(stderr,"Unable to %s",getCmd);
        exit(1);
    }

    time(&time_finished);
    char starttime_str[256];
    char endtime_str[256];
    struct tm * localstart = localtime( &time_started );
    struct tm * localend = localtime ( &time_finished );
    strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
    strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
    sprintf(getCmd,"set treewalk_timestamp \"%s\"",endtime_str);
    if(!benchmarking_flag && redis_blocking_command(getCmd,NULL,INT) < 0)
    {
        fprintf(stderr,"Unable to %s",getCmd);
    }
    
    if(rank == 0)
    {
        LOG(PURGER_LOG_INFO, "treewalk run started at: %s", starttime_str);
        LOG(PURGER_LOG_INFO, "treewalk run completed at: %s", endtime_str);
        LOG(PURGER_LOG_INFO, "treewalk total time (seconds) for this run: %f",difftime(time_finished,time_started));
        LOG(PURGER_LOG_INFO, "\nTotal time in process_objects: %lf\n\
                   \tRedis commands: %lf %lf\n\
                   \tStating:  %lf %lf\n\
                   \tReaddir: %lf %lf\n\
                   \tHashing: %lf %lf\n",
                   process_objects_total[1],redis_time[1],redis_time[1]/process_objects_total[1]*100.0,stat_time[1],stat_time[1]/process_objects_total[1]*100.0,readdir_time[1],readdir_time[1]/process_objects_total[1]*100.0
                   ,hash_time[1],hash_time[1]/process_objects_total[1]*100.0);
    }
コード例 #6
0
ファイル: mfu_flist_archive.c プロジェクト: hpc/fileutils
static void mfu_flist_archive_create_libcircle(mfu_flist flist, const char* archivefile, mfu_archive_options_t* opts)
{
    DTAR_flist = flist;
    DTAR_user_opts = *opts;

    MPI_Comm_rank(MPI_COMM_WORLD, &DTAR_rank);

    /* TODO: stripe the archive file if on parallel file system */

    /* init statistics */
    DTAR_statistics.total_dirs  = 0;
    DTAR_statistics.total_files = 0;
    DTAR_statistics.total_links = 0;
    DTAR_statistics.total_size  = 0;
    DTAR_statistics.total_bytes_copied = 0;

    time(&(DTAR_statistics.time_started));
    DTAR_statistics.wtime_started = MPI_Wtime();

    /* create the archive file */
    DTAR_writer.name = archivefile;
    DTAR_writer.flags = O_WRONLY | O_CREAT | O_CLOEXEC | O_LARGEFILE;
    DTAR_writer.fd_tar = open(archivefile, DTAR_writer.flags, 0664);

    /* get number of items in our portion of the list */
    DTAR_count = mfu_flist_size(DTAR_flist);

    /* allocate memory for file sizes and offsets */
    uint64_t* fsizes = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
    DTAR_offsets     = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));

    /* compute local offsets for each item and total
     * bytes we're contributing to the archive */
    uint64_t idx;
    uint64_t offset = 0;
    for (idx = 0; idx < DTAR_count; idx++) {
        /* assume the item takes no space */
        fsizes[idx] = 0;

        /* identify item type to compute its size in the archive */
        mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
        if (type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
            /* directories and symlinks only need the header */
            fsizes[idx] = DTAR_HDR_LENGTH;
        } else if (type == MFU_TYPE_FILE) {
            /* regular file requires a header, plus file content,
             * and things are packed into blocks of 512 bytes */
            uint64_t fsize = mfu_flist_file_get_size(DTAR_flist, idx);

            /* determine whether file size is integer multiple of 512 bytes */
            uint64_t rem = fsize % 512;
            if (rem == 0) {
                /* file content is multiple of 512 bytes, so perfect fit */
                fsizes[idx] = fsize + DTAR_HDR_LENGTH;
            } else {
                /* TODO: check and explain this math */
                fsizes[idx] = (fsize / 512 + 4) * 512;
            }

        }

        /* increment our local offset for this item */
        DTAR_offsets[idx] = offset;
        offset += fsizes[idx];
    }

    /* execute scan to figure our global base offset in the archive file */
    uint64_t global_offset = 0;
    MPI_Scan(&offset, &global_offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
    global_offset -= offset;

    /* update offsets for each of our file to their global offset */
    for (idx = 0; idx < DTAR_count; idx++) {
        DTAR_offsets[idx] += global_offset;
    }

    /* create an archive */
    struct archive* ar = archive_write_new();

    archive_write_set_format_pax(ar);

    int r = archive_write_open_fd(ar, DTAR_writer.fd_tar);
    if (r != ARCHIVE_OK) {
        MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
        DTAR_abort(EXIT_FAILURE);
    }

    /* write headers for our files */
    for (idx = 0; idx < DTAR_count; idx++) {
        mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
        if (type == MFU_TYPE_FILE || type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
            DTAR_write_header(ar, idx, DTAR_offsets[idx]);
        }
    }

    /* prepare libcircle */
    CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL);
    CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
    CIRCLE_enable_logging(loglevel);

    /* register callbacks */
    CIRCLE_cb_create(&DTAR_enqueue_copy);
    CIRCLE_cb_process(&DTAR_perform_copy);

    /* run the libcircle job to copy data into archive file */
    CIRCLE_begin();
    CIRCLE_finalize();

    /* compute total bytes copied */
    uint64_t archive_size = 0;
    MPI_Allreduce(&offset, &archive_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
    DTAR_statistics.total_size = archive_size;

    DTAR_statistics.wtime_ended = MPI_Wtime();
    time(&(DTAR_statistics.time_ended));

    /* print stats */
    double rel_time = DTAR_statistics.wtime_ended - \
                      DTAR_statistics.wtime_started;
    if (DTAR_rank == 0) {
        char starttime_str[256];
        struct tm* localstart = localtime(&(DTAR_statistics.time_started));
        strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart);

        char endtime_str[256];
        struct tm* localend = localtime(&(DTAR_statistics.time_ended));
        strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend);

        /* add two 512 blocks at the end */
        DTAR_statistics.total_size += 512 * 2;

        /* convert bandwidth to unit */
        double agg_rate_tmp;
        double agg_rate = (double) DTAR_statistics.total_size / rel_time;
        const char* agg_rate_units;
        mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units);

        MFU_LOG(MFU_LOG_INFO, "Started:    %s", starttime_str);
        MFU_LOG(MFU_LOG_INFO, "Completed:  %s", endtime_str);
        MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size);
        MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \
                "(%.3" PRIu64 " bytes in %.3lf seconds)", \
                agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time);
    }

    /* clean up */
    mfu_free(&fsizes);
    mfu_free(&DTAR_offsets);

    /* close archive file */
    archive_write_free(ar);
    mfu_close(DTAR_writer.name, DTAR_writer.fd_tar);
}
コード例 #7
0
ファイル: warnusers.c プロジェクト: jlafon/purger
int
main (int argc, char **argv)
{
    int index;
    int c;

    char *redis_hostname;
    int redis_port;

    int redis_hostname_flag = 0;
    int redis_port_flag = 0;
    int sharded_flag = 0;
    int ldap_flag = 0;
    int force_flag = 0;
    int db_number = 0;
    sharded_count = 0;
    char * redis_host_list;
    redis_command_ptr = &non_sharded_redis_command;
    PURGER_debug_stream = stdout;
    PURGER_debug_level = PURGER_LOG_DBG;

    int PURGER_global_rank = CIRCLE_init(argc, argv);
    if(PURGER_global_rank < 0)
        exit(1);
    opterr = 0;
    CIRCLE_enable_logging(CIRCLE_LOG_ERR);
    while((c = getopt(argc, argv, "h:p:l:fi:s:a:o:")) != -1)
    {
        switch(c)
        {
            case 'o':
                ldap_port = atoi(optarg);
                break;
            case 'a':
                ldap_server = optarg;
                ldap_flag = 1;
                break;
            case 'h':
                redis_hostname = optarg;
                redis_hostname_flag = 1;
                break;

            case 'i':
                db_number = atoi(optarg);
                break;

            case 'p':
                redis_port = atoi(optarg);
                redis_port_flag = 1;
                break;

            case 'f':
                force_flag = 1;
                if(PURGER_global_rank == 0) LOG(PURGER_LOG_WARN,"Warning: You have chosen to force warnusers.");
                break; 
            case 'l':
                PURGER_debug_level = atoi(optarg);
                break;
            
            case 's':
                redis_host_list = optarg;
                sharded_flag = 1;
                redis_command_ptr = &redis_blocking_shard_command;
                break;

            case '?':
                if (PURGER_global_rank == 0 && (optopt == 'h' || optopt == 'p' || optopt == 'l'))
                {
                    print_usage(argv);
                    fprintf(stderr, "Option -%c requires an argument.\n", optopt);
                    exit(EXIT_FAILURE);
                }
                else if (isprint (optopt))
                {
                    print_usage(argv);
                    fprintf(stderr, "Unknown option `-%c'.\n", optopt);
                    exit(EXIT_FAILURE);
                }
                else
                {
                    print_usage(argv);
                    fprintf(stderr,
                        "Unknown option character `\\x%x'.\n",
                        optopt);
                    exit(EXIT_FAILURE);
                }

            default:
                abort();
        }
    }


    if(redis_hostname_flag == 0)
    {
        if(PURGER_global_rank == 0) LOG(PURGER_LOG_WARN, "A hostname for redis was not specified, defaulting to localhost.");
        redis_hostname = "localhost";
    }

    if(redis_port_flag == 0)
    {
        if(PURGER_global_rank == 0) LOG(PURGER_LOG_WARN, "A port number for redis was not specified, defaulting to 6379.");
        redis_port = 6379;
    }

    for (index = optind; index < argc; index++)
        LOG(PURGER_LOG_WARN, "Non-option argument %s", argv[index]);

    if(redis_init(redis_hostname, redis_port, db_number) < 0)
    {
        LOG(PURGER_LOG_ERR,"Unable to initialize redis.");
        exit(0);
    }
    if(sharded_flag)
    {
        LOG(PURGER_LOG_INFO,"Sharding enabled.");
        sharded_count = redis_shard_init(redis_host_list, redis_port, db_number);
    }
    if(!ldap_flag)
    {
        LOG(PURGER_LOG_ERR,"You must specify an ldap server.");
        exit(EXIT_FAILURE);
    }
    else
        ldap_init(ldap_server,ldap_port); 
    mailinfo.from     = "*****@*****.**";
    mailinfo.fromreal = "*****@*****.**";
    mailinfo.subject = "[PURGER NOTIFICATION]";
    mailinfo.defaultto = "*****@*****.**";
    mailinfo.server = "mail.lanl.gov";
    mailinfo.txt = "The following text file in the Turquiose network contains a list of       \
                   your scratch files that have not been modified in the last 14+ days.       \
                   Those files will be deleted in at least 6 days if not modified by then.    \
                   This notification may not have up-to-the-minute information, but we        \
                   will verify a file's actual age before purging it.   For more information, \
                    please see our purge policy:  http://hpc.lanl.gov/purge_policy.           \
                   If you have questions or concerns, please contact ICN Consultants          \
                   at 505-665-4444 option 3."; 

    time(&time_started);
    if(warnusers_check_state(PURGER_global_rank,force_flag) < 0)
        exit(1);
    CIRCLE_cb_create(&add_objects);
    CIRCLE_cb_process(&process_objects);
    CIRCLE_begin();
    CIRCLE_finalize();
    time(&time_finished);
    char starttime_str[256];
    char endtime_str[256];
    struct tm * localstart = localtime( &time_started );
    struct tm * localend = localtime ( &time_finished );
    strftime(starttime_str, 256, "%b-%d-%Y,%H:%M:%S",localstart);
    strftime(endtime_str, 256, "%b-%d-%Y,%H:%M:%S",localend);
    char getCmd[256];
    sprintf(getCmd,"set warnusers_timestamp \"%s\"",endtime_str);
    if(redis_blocking_command(getCmd,(void*)NULL,NIL)<0)
    {   
         LOG(PURGER_LOG_ERR,"Unable to %s",getCmd);
    }  
    sprintf(getCmd,"set warnusers-rank-%d 0", PURGER_global_rank);
    if(redis_blocking_command(getCmd,(void*)NULL,NIL)<0)
    {
         LOG(PURGER_LOG_ERR,"Unable to %s",getCmd);
    }
    redis_finalize();
    if(sharded_flag)
        redis_shard_finalize();
    if(PURGER_global_rank == 0)
    {
        LOG(PURGER_LOG_INFO, "Warnusers run started at: %s", starttime_str);
        LOG(PURGER_LOG_INFO, "Warnusers run completed at: %s", endtime_str);
        LOG(PURGER_LOG_INFO, "Warnusers total time (seconds) for this run: %f",difftime(time_finished,time_started));
    }
    exit(EXIT_SUCCESS);
}
コード例 #8
0
ファイル: dcp.c プロジェクト: bringhurst/dcp
int main(int argc, \
         char** argv)
{
    int c;
    int option_index = 0;

    MPI_Init(&argc, &argv);

    /* Initialize our processing library and related callbacks. */
    /* This is a bit of chicken-and-egg problem, because we'd like
     * to have our rank to filter output messages below but we might
     * also want to set different libcircle flags based on command line
     * options -- for now just pass in the default flags */
    CIRCLE_global_rank = CIRCLE_init(argc, argv, CIRCLE_DEFAULT_FLAGS);
    CIRCLE_cb_create(&DCOPY_add_objects);
    CIRCLE_cb_process(&DCOPY_process_objects);

    DCOPY_debug_stream = stdout;

    /* By default, don't perform a conditional copy. */
    DCOPY_user_opts.conditional = false;

    /* By default, don't skip the compare option. */
    DCOPY_user_opts.skip_compare = false;

    /* By default, show info log messages. */
    CIRCLE_loglevel CIRCLE_debug = CIRCLE_LOG_INFO;
    DCOPY_debug_level = DCOPY_LOG_DBG;

    /* By default, don't unlink destination files if an open() fails. */
    DCOPY_user_opts.force = false;

    /* By default, don't bother to preserve all attributes. */
    DCOPY_user_opts.preserve = false;

    /* By default, don't attempt any type of recursion. */
    DCOPY_user_opts.recursive = false;
    DCOPY_user_opts.recursive_unspecified = false;

    /* By default, assume the filesystem is reliable (exit on errors). */
    DCOPY_user_opts.reliable_filesystem = true;

    static struct option long_options[] = {
        {"conditional"          , no_argument      , 0, 'c'},
        {"skip-compare"         , no_argument      , 0, 'C'},
        {"debug"                , required_argument, 0, 'd'},
        {"force"                , no_argument      , 0, 'f'},
        {"help"                 , no_argument      , 0, 'h'},
        {"preserve"             , no_argument      , 0, 'p'},
        {"recursive"            , no_argument      , 0, 'R'},
        {"recursive-unspecified", no_argument      , 0, 'r'},
        {"unreliable-filesystem", no_argument      , 0, 'U'},
        {"version"              , no_argument      , 0, 'v'},
        {0                      , 0                , 0, 0  }
    };

    /* Parse options */
    while((c = getopt_long(argc, argv, "cCd:fhpRrUv", \
                           long_options, &option_index)) != -1) {
        switch(c) {

            case 'c':
                DCOPY_user_opts.conditional = true;

                if(CIRCLE_global_rank == 0) {
                    LOG(DCOPY_LOG_INFO, "Performing a conditional copy.");
                }

                break;

            case 'C':
                DCOPY_user_opts.skip_compare = true;

                if(CIRCLE_global_rank == 0) {
                    LOG(DCOPY_LOG_INFO, "Skipping the comparison stage " \
                        "(may result in corruption).");
                }

                break;

            case 'd':

                if(strncmp(optarg, "fatal", 5)) {
                    CIRCLE_debug = CIRCLE_LOG_FATAL;
                    DCOPY_debug_level = DCOPY_LOG_FATAL;

                    if(CIRCLE_global_rank == 0) {
                        LOG(DCOPY_LOG_INFO, "Debug level set to: fatal");
                    }

                }
                else if(strncmp(optarg, "err", 3)) {
                    CIRCLE_debug = CIRCLE_LOG_ERR;
                    DCOPY_debug_level = DCOPY_LOG_ERR;

                    if(CIRCLE_global_rank == 0) {
                        LOG(DCOPY_LOG_INFO, "Debug level set to: errors");
                    }

                }
                else if(strncmp(optarg, "warn", 4)) {
                    CIRCLE_debug = CIRCLE_LOG_WARN;
                    DCOPY_debug_level = DCOPY_LOG_WARN;

                    if(CIRCLE_global_rank == 0) {
                        LOG(DCOPY_LOG_INFO, "Debug level set to: warnings");
                    }

                }
                else if(strncmp(optarg, "info", 4)) {
                    CIRCLE_debug = CIRCLE_LOG_INFO;
                    DCOPY_debug_level = DCOPY_LOG_INFO;

                    if(CIRCLE_global_rank == 0) {
                        LOG(DCOPY_LOG_INFO, "Debug level set to: info");
                    }

                }
                else if(strncmp(optarg, "dbg", 4)) {
                    CIRCLE_debug = CIRCLE_LOG_DBG;
                    DCOPY_debug_level = DCOPY_LOG_DBG;

                    if(CIRCLE_global_rank == 0) {
                        LOG(DCOPY_LOG_INFO, "Debug level set to: debug");
                    }

                }
                else {
                    if(CIRCLE_global_rank == 0) {
                        LOG(DCOPY_LOG_INFO, "Debug level `%s' not recognized. " \
                            "Defaulting to `info'.", optarg);
                    }
                }

                break;

            case 'f':
                DCOPY_user_opts.force = true;

                if(CIRCLE_global_rank == 0) {
                    LOG(DCOPY_LOG_INFO, "Deleting destination on errors.");
                }

                break;

            case 'h':

                if(CIRCLE_global_rank == 0) {
                    DCOPY_print_usage(argv);
                }

                DCOPY_exit(EXIT_SUCCESS);
                break;

            case 'p':
                DCOPY_user_opts.preserve = true;

                if(CIRCLE_global_rank == 0) {
                    LOG(DCOPY_LOG_INFO, "Preserving file attributes.");
                }

                break;

            case 'R':
                DCOPY_user_opts.recursive = true;

                if(CIRCLE_global_rank == 0) {
                    LOG(DCOPY_LOG_INFO, "Performing correct recursion.");
                    LOG(DCOPY_LOG_WARN, "Warning, only files and directories are implemented.");
                }

                break;

            case 'r':
                DCOPY_user_opts.recursive_unspecified = true;

                if(CIRCLE_global_rank == 0) {
                    LOG(DCOPY_LOG_INFO, "Performing recursion. " \
                        "Ignoring special files.");
                }

                break;

            case 'U':
                DCOPY_user_opts.reliable_filesystem = false;

                if(CIRCLE_global_rank == 0) {
                    LOG(DCOPY_LOG_INFO, "Unreliable filesystem specified. " \
                        "Retry mode enabled.");
                }

                break;

            case 'v':

                if(CIRCLE_global_rank == 0) {
                    DCOPY_print_version();
                }

                DCOPY_exit(EXIT_SUCCESS);
                break;

            case '?':
            default:

                if(CIRCLE_global_rank == 0) {
                    if(optopt == 'd') {
                        DCOPY_print_usage(argv);
                        fprintf(stderr, "Option -%c requires an argument.\n", \
                                optopt);
                    }
                    else if(isprint(optopt)) {
                        DCOPY_print_usage(argv);
                        fprintf(stderr, "Unknown option `-%c'.\n", optopt);
                    }
                    else {
                        DCOPY_print_usage(argv);
                        fprintf(stderr,
                                "Unknown option character `\\x%x'.\n",
                                optopt);
                    }
                }

                DCOPY_exit(EXIT_FAILURE);
                break;
        }
    }

    /** Parse the source and destination paths. */
    DCOPY_parse_path_args(argv, optind, argc);

    /* initialize linked list of stat objects */
    DCOPY_list_head  = NULL;
    DCOPY_list_tail  = NULL;

    /* Initialize our jump table for core operations. */
    DCOPY_jump_table[TREEWALK] = DCOPY_do_treewalk;
    DCOPY_jump_table[COPY]     = DCOPY_do_copy;
    DCOPY_jump_table[CLEANUP]  = DCOPY_do_cleanup;
    DCOPY_jump_table[COMPARE]  = DCOPY_do_compare;

    /* Set the log level for the processing library. */
    CIRCLE_enable_logging(CIRCLE_debug);

    /* Grab a relative and actual start time for the epilogue. */
    time(&(DCOPY_statistics.time_started));
    DCOPY_statistics.wtime_started = CIRCLE_wtime();

    /* Perform the actual file copy. */
    CIRCLE_begin();

    /* Determine the actual and relative end time for the epilogue. */
    DCOPY_statistics.wtime_ended = CIRCLE_wtime();
    time(&(DCOPY_statistics.time_ended));

    /* Let the processing library cleanup. */
    CIRCLE_finalize();

    /* set timestamps if needed */
    if (DCOPY_user_opts.preserve) {
        DCOPY_set_timestamps();
    }

    /* free list of stat objects */
    DCOPY_stat_elem_t* current = DCOPY_list_head;
    while (current != NULL) {
        DCOPY_stat_elem_t* next = current->next;
        free(current->file);
        free(current->sb);
        free(current);
        current = next;
    }

    /* Print the results to the user. */
    DCOPY_epilogue();

    DCOPY_exit(EXIT_SUCCESS);
}