int EntryProc_db_flag_op( struct entry_proc_op_t *p_op, lmgr_t * lmgr ) { int rc; /* insert to DB */ switch ( p_op->db_op_type ) { attr_set_t attr_chg; lmgr_filter_t filter; filter_value_t val; case OP_TYPE_UPDATE: /* set previous_scan flag */ ATTR_MASK_INIT( &attr_chg ); ATTR_MASK_SET( &attr_chg, previous_scan ); ATTR( &attr_chg, previous_scan ) = TRUE; rc = ListMgr_MassUpdate( lmgr, NULL, &attr_chg ); break; case OP_TYPE_REMOVE: val.val_bool = TRUE; lmgr_simple_filter_init( &filter ); lmgr_simple_filter_add( &filter, ATTR_INDEX_previous_scan, EQUAL, val ); /* remove entries with previous_scan flag */ rc = ListMgr_MassRemove( lmgr, &filter ); lmgr_simple_filter_free( &filter ); break; default: printf( "unhandled\n" ); rc = -1; } if ( rc ) printf( "ERROR: ListMgr mass operation returned %d\n", rc ); if ( p_op->callback_func ) p_op->callback_func( lmgr, p_op, p_op->callback_param ); rc = EntryProcessor_Acknowledge( p_op, 0, TRUE ); return rc; }
/** * Main daemon routine */ int main( int argc, char **argv ) { int c, i, option_index = 0; char *bin = basename( argv[0] ); int rc; char err_msg[4096]; robinhood_config_t rh_config; int chgd = 0; char badcfg[RBH_PATH_MAX]; char tag_name[256] = ""; start_time = time( NULL ); zero_options( &options ); /* parse command line options */ while ( ( c = getopt_long( argc, argv, SHORT_OPT_STRING, option_tab, &option_index ) ) != -1 ) { switch ( c ) { case 's': options.partial_scan = TRUE; rh_strncpy(options.partial_scan_path, optarg, RBH_PATH_MAX); /* clean final slash */ if (FINAL_SLASH(options.partial_scan_path)) REMOVE_FINAL_SLASH(options.partial_scan_path); break; case 'd': if (parse_diff_mask(optarg, &options.diff_mask, err_msg)) { fprintf(stderr, "Invalid argument for --diff: %s\n", err_msg); exit( 1 ); } break; case 'a': if (optarg) { if (!strcasecmp(optarg,"fs")) options.diff_arg.apply = APPLY_FS; else if (!strcasecmp(optarg,"db")) options.diff_arg.apply = APPLY_DB; else { fprintf(stderr, "Invalid argument for --apply: '%s' (fs or db expected)\n", optarg); exit( 1 ); } } else options.diff_arg.apply = APPLY_DB; break; case 'D': options.flags |= FLAG_DRY_RUN; break; case 'f': rh_strncpy(options.config_file, optarg, MAX_OPT_LEN); break; #ifdef _HSM_LITE case 'b': options.diff_arg.recov_from_backend = 1; break; #endif #ifdef _HAVE_FID /* only for lustre 2.x */ case 'o': rh_strncpy(options.output_dir, optarg, MAX_OPT_LEN); break; #endif case 'l': options.force_log_level = TRUE; options.log_level = str2debuglevel( optarg ); if ( options.log_level == -1 ) { fprintf( stderr, "Unsupported log level '%s'. CRIT, MAJOR, EVENT, VERB, DEBUG or FULL expected.\n", optarg ); exit( 1 ); } break; case 'h': display_help( bin ); exit( 0 ); break; case 'V': display_version( bin ); exit( 0 ); break; case ':': case '?': default: fprintf(stderr,"Run '%s --help' for more details.\n", bin); exit( 1 ); break; } } /* check there is no extra arguments */ if ( optind != argc ) { fprintf( stderr, "Error: unexpected argument on command line: %s\n", argv[optind] ); exit( 1 ); } /* Initialize global tools */ #ifdef _LUSTRE if ( ( rc = Lustre_Init( ) ) ) { fprintf( stderr, "Error %d initializing liblustreapi\n", rc ); exit( 1 ); } #endif /* Initilize uidgid cache */ if ( InitUidGid_Cache( ) ) { fprintf( stderr, "Error initializing uid/gid cache\n" ); exit( 1 ); } /* get default config file, if not specified */ if (SearchConfig(options.config_file, options.config_file, &chgd, badcfg, MAX_OPT_LEN) != 0) { fprintf(stderr, "No config file (or too many) found matching %s\n", badcfg); exit(2); } else if (chgd) { fprintf(stderr, "Using config file '%s'.\n", options.config_file ); } if ( ReadRobinhoodConfig( MODULE_MASK_FS_SCAN | MODULE_MASK_ENTRY_PROCESSOR, options.config_file, err_msg, &rh_config, FALSE ) ) { fprintf( stderr, "Error reading configuration file '%s': %s\n", options.config_file, err_msg ); exit( 1 ); } process_config_file = options.config_file; /* set global configuration */ global_config = rh_config.global_config; /* set policies info */ policies = rh_config.policies; if (options.force_log_level) rh_config.log_config.debug_level = options.log_level; else rh_config.log_config.debug_level = LVL_CRIT; /* no event message */ /* Set logging to stderr */ strcpy( rh_config.log_config.log_file, "stderr" ); strcpy( rh_config.log_config.report_file, "stderr" ); strcpy( rh_config.log_config.alert_file, "stderr" ); /* Initialize logging */ rc = InitializeLogs( bin, &rh_config.log_config ); if ( rc ) { fprintf( stderr, "Error opening log files: rc=%d, errno=%d: %s\n", rc, errno, strerror( errno ) ); exit( rc ); } /* Initialize filesystem access */ rc = InitFS(); if (rc) exit(rc); #ifdef _HSM_LITE rc = Backend_Start( &rh_config.backend_config, options.flags ); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error initializing backend" ); exit( 1 ); } #endif /* Initialize list manager */ rc = ListMgr_Init( &rh_config.lmgr_config, FALSE ); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error %d initializing list manager", rc ); exit( rc ); } else DisplayLog( LVL_VERB, DIFF_TAG, "ListManager successfully initialized" ); if ( CheckLastFS( ) != 0 ) exit( 1 ); if (options.diff_mask) rh_config.entry_proc_config.diff_mask = options.diff_mask; else { /* parse "all" */ char tmpstr[] = "all"; if (parse_diff_mask(tmpstr, &rh_config.entry_proc_config.diff_mask, err_msg)) { DisplayLog(LVL_CRIT, DIFF_TAG, "unexpected error parsing diff mask: %s", err_msg); exit(1); } } #ifdef LUSTRE_DUMP_FILES if (options.diff_arg.apply == APPLY_FS && !(options.flags & FLAG_DRY_RUN)) { /* open the file to write LOV EA and FID remapping */ if (!EMPTY_STRING(options.output_dir)) { char fname[RBH_PATH_MAX]; if (mkdir(options.output_dir, 0700) && (errno != EEXIST)) { DisplayLog(LVL_CRIT, DIFF_TAG, "Failed to create directory %s: %s", options.output_dir, strerror(errno)); exit(1); } snprintf(fname, RBH_PATH_MAX-1, "%s/"LOVEA_FNAME, options.output_dir); options.diff_arg.lovea_file = fopen(fname, "w"); if (options.diff_arg.lovea_file == NULL) { DisplayLog(LVL_CRIT, DIFF_TAG, "Failed to open %s for writting: %s", fname, strerror(errno)); exit(1); } snprintf(fname, RBH_PATH_MAX-1, "%s/"FIDREMAP_FNAME, options.output_dir); options.diff_arg.fid_remap_file = fopen(fname, "w"); if (options.diff_arg.fid_remap_file == NULL) { DisplayLog(LVL_CRIT, DIFF_TAG, "Failed to open %s for writting: %s", fname, strerror(errno)); exit(1); } } } #endif /* if no DB apply action is specified, can't use md_update field for checking * removed entries. So, create a special tag for that. */ if ((options.diff_arg.apply != APPLY_DB) || (options.flags & FLAG_DRY_RUN)) { fprintf(stderr, "Preparing diff table...\n"); /* create a connexion to the DB. this is safe to use the global lmgr var * as statistics thread is not running */ if (!ensure_db_access()) exit(1); /* create a tag to clear entries after the scan */ /* There could be several diff running in parallel, * so set a suffix to avoid conflicts */ sprintf(tag_name, "DIFF_%u", (unsigned int) getpid()); options.diff_arg.db_tag = tag_name; /* add filter for partial scan */ if (options.partial_scan) { lmgr_filter_t filter; filter_value_t val; lmgr_simple_filter_init( &filter ); char tmp[RBH_PATH_MAX]; strcpy(tmp, options.partial_scan_path); strcat(tmp, "/*"); val.value.val_str = tmp; lmgr_simple_filter_add(&filter, ATTR_INDEX_fullpath, LIKE, val, 0); rc = ListMgr_CreateTag(&lmgr, tag_name, &filter, FALSE); lmgr_simple_filter_free(&filter); } else rc = ListMgr_CreateTag(&lmgr, tag_name, NULL, FALSE); if (rc) exit(rc); } /* Initialise Pipeline */ rc = EntryProcessor_Init(&rh_config.entry_proc_config, DIFF_PIPELINE, options.flags, &options.diff_arg); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error %d initializing EntryProcessor pipeline", rc ); goto clean_tag; } else DisplayLog( LVL_VERB, DIFF_TAG, "EntryProcessor successfully initialized" ); fprintf(stderr, "Starting scan\n"); /* print header to indicate the content of diff * #<diff cmd> * ---fs[=/subdir] * +++db */ for (i = 0; i < argc; i++) printf("%s%s", i==0?"# ":" ", argv[i]); printf("\n"); if (options.diff_arg.apply == APPLY_FS) { if (options.partial_scan) printf("---fs=%s\n",options.partial_scan_path); else printf("---fs\n"); printf("+++db\n"); } else { printf("---db\n"); if (options.partial_scan) printf("+++fs=%s\n",options.partial_scan_path); else printf("+++fs\n"); } /* Start FS scan */ if (options.partial_scan) rc = FSScan_Start(&rh_config.fs_scan_config, options.flags, options.partial_scan_path); else rc = FSScan_Start(&rh_config.fs_scan_config, options.flags, NULL); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error %d initializing FS Scan module", rc ); goto clean_tag; } else DisplayLog( LVL_VERB, DIFF_TAG, "FS Scan module successfully initialized" ); /* Flush logs now, to have a trace in the logs */ FlushLogs( ); /* both pipeline and scan are now running, can now trap events and display stats */ /* create signal handling thread */ rc = pthread_create( &sig_thr, NULL, signal_handler_thr, NULL ); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error starting signal handler thread: %s", strerror( errno ) ); goto clean_tag; } else DisplayLog( LVL_VERB, DIFF_TAG, "Signal handler thread started successfully" ); pthread_create(&stat_thread, NULL, stats_thr, NULL); /* wait for FS scan to end */ FSScan_Wait( ); DisplayLog( LVL_MAJOR, DIFF_TAG, "FS Scan finished" ); /* Pipeline must be flushed */ EntryProcessor_Terminate( TRUE ); #ifdef LUSTRE_DUMP_FILES /* flush the lovea file */ if (options.diff_arg.lovea_file) { fprintf(stderr, " > LOV EA information written to %s/"LOVEA_FNAME"\n", options.output_dir); fclose(options.diff_arg.lovea_file); } if (options.diff_arg.fid_remap_file) { fprintf(stderr, " > FID remapping written to %s/"FIDREMAP_FNAME"\n", options.output_dir); fclose(options.diff_arg.fid_remap_file); } #endif fprintf(stderr, "End of scan\n"); DisplayLog( LVL_MAJOR, DIFF_TAG, "All tasks done! Exiting." ); rc = 0; clean_tag: /* destroy the tag before exit */ if (options.diff_arg.db_tag != NULL && ensure_db_access()) { fprintf(stderr, "Cleaning diff table...\n"); ListMgr_DestroyTag(&lmgr, options.diff_arg.db_tag); } exit(rc); return rc; /* for compiler */ }
/** scan sets of directories * \param cb_func, callback function for each set of directory */ int rbh_scrub(lmgr_t *p_mgr, const wagon_t *id_list, unsigned int id_count, attr_mask_t dir_attr_mask, scrub_callback_t cb_func, void *arg) { wagon_t *curr_array; unsigned int count; lmgr_filter_t filter; filter_value_t fv; int i, rc; int last_err = 0; rc = add_id_list(id_list, id_count); if (rc) return rc; /* only get subdirs (for scanning) */ fv.value.val_str = STR_TYPE_DIR; lmgr_simple_filter_init(&filter); lmgr_simple_filter_add(&filter, ATTR_INDEX_type, EQUAL, fv, 0); /* while the array is not empty */ while (array_used > 0) { unsigned int res_count = 0; wagon_t *child_ids; attr_set_t *child_attrs; /* get a set of entry_ids */ curr_array = &dir_array[array_first]; if (array_used < LS_CHUNK) { /* get all available dirs */ count = array_used; } else { /* get a constant chunk */ count = LS_CHUNK; } #ifdef _DEBUG_ID_LIST printf("processing %u-%u\n", array_first, array_first + count - 1); #endif /* read childs */ res_count = 0; child_ids = NULL; child_attrs = NULL; rc = ListMgr_GetChild(p_mgr, &filter, curr_array, count, dir_attr_mask, &child_ids, &child_attrs, &res_count); if (rc) { DisplayLog(LVL_CRIT, SCRUB_TAG, "ListMgr_GetChild() terminated with error %d", rc); /* @TODO free allocated resources */ break; } /* Call the callback func for each listed dir */ rc = cb_func(child_ids, child_attrs, res_count, arg); if (rc) /* XXX break the scan? */ last_err = rc; /* can release the list of input ids */ rbh_scrub_release_list(array_first, count); /* copy entry ids before freeing them */ /* TODO: we could transfer the pathname instead of strdup() them. */ add_id_list(child_ids, res_count); /* attributes no longer needed */ /* release attrs */ if (child_attrs) { for (i = 0; i < res_count; i++) ListMgr_FreeAttrs(&child_attrs[i]); MemFree(child_attrs); child_attrs = NULL; } /* free the returned id array */ if (child_ids) { free_wagon(child_ids, 0, res_count); MemFree(child_ids); child_ids = NULL; } } lmgr_simple_filter_free(&filter); return last_err; }