static void dump_stats( lmgr_t * lmgr ) { char tmp_buff[256]; time_t now; struct tm date; now = time( NULL ); strftime( tmp_buff, 256, "%Y/%m/%d %T", localtime_r( &now, &date ) ); DisplayLog( LVL_MAJOR, "STATS", "==================== Dumping stats at %s =====================", tmp_buff ); DisplayLog( LVL_MAJOR, "STATS", "Diff start time: %s", start_time_str ); FSScan_DumpStats(); EntryProcessor_DumpCurrentStages(); /* Flush stats */ FlushLogs( ); }
void Exit( int error_code ) { DisplayLog( LVL_MAJOR, "EXIT", "Exiting program with code %d", error_code ); #if 0 /* Remove pid_file, if any */ if ( pid_file != NULL ) { if ( unlink( pid_file ) != 0 ) { DisplayLog( LVL_CRIT, "EXIT", "Could not remove pid file %s: %s", pid_file, strerror( errno ) ); } } #endif FlushLogs( ); exit( error_code ); }
/** * Main daemon routine */ int main( int argc, char **argv ) { int c, i, option_index = 0; char *bin = basename( argv[0] ); int rc; char err_msg[4096]; robinhood_config_t rh_config; int chgd = 0; char badcfg[RBH_PATH_MAX]; char tag_name[256] = ""; start_time = time( NULL ); zero_options( &options ); /* parse command line options */ while ( ( c = getopt_long( argc, argv, SHORT_OPT_STRING, option_tab, &option_index ) ) != -1 ) { switch ( c ) { case 's': options.partial_scan = TRUE; rh_strncpy(options.partial_scan_path, optarg, RBH_PATH_MAX); /* clean final slash */ if (FINAL_SLASH(options.partial_scan_path)) REMOVE_FINAL_SLASH(options.partial_scan_path); break; case 'd': if (parse_diff_mask(optarg, &options.diff_mask, err_msg)) { fprintf(stderr, "Invalid argument for --diff: %s\n", err_msg); exit( 1 ); } break; case 'a': if (optarg) { if (!strcasecmp(optarg,"fs")) options.diff_arg.apply = APPLY_FS; else if (!strcasecmp(optarg,"db")) options.diff_arg.apply = APPLY_DB; else { fprintf(stderr, "Invalid argument for --apply: '%s' (fs or db expected)\n", optarg); exit( 1 ); } } else options.diff_arg.apply = APPLY_DB; break; case 'D': options.flags |= FLAG_DRY_RUN; break; case 'f': rh_strncpy(options.config_file, optarg, MAX_OPT_LEN); break; #ifdef _HSM_LITE case 'b': options.diff_arg.recov_from_backend = 1; break; #endif #ifdef _HAVE_FID /* only for lustre 2.x */ case 'o': rh_strncpy(options.output_dir, optarg, MAX_OPT_LEN); break; #endif case 'l': options.force_log_level = TRUE; options.log_level = str2debuglevel( optarg ); if ( options.log_level == -1 ) { fprintf( stderr, "Unsupported log level '%s'. CRIT, MAJOR, EVENT, VERB, DEBUG or FULL expected.\n", optarg ); exit( 1 ); } break; case 'h': display_help( bin ); exit( 0 ); break; case 'V': display_version( bin ); exit( 0 ); break; case ':': case '?': default: fprintf(stderr,"Run '%s --help' for more details.\n", bin); exit( 1 ); break; } } /* check there is no extra arguments */ if ( optind != argc ) { fprintf( stderr, "Error: unexpected argument on command line: %s\n", argv[optind] ); exit( 1 ); } /* Initialize global tools */ #ifdef _LUSTRE if ( ( rc = Lustre_Init( ) ) ) { fprintf( stderr, "Error %d initializing liblustreapi\n", rc ); exit( 1 ); } #endif /* Initilize uidgid cache */ if ( InitUidGid_Cache( ) ) { fprintf( stderr, "Error initializing uid/gid cache\n" ); exit( 1 ); } /* get default config file, if not specified */ if (SearchConfig(options.config_file, options.config_file, &chgd, badcfg, MAX_OPT_LEN) != 0) { fprintf(stderr, "No config file (or too many) found matching %s\n", badcfg); exit(2); } else if (chgd) { fprintf(stderr, "Using config file '%s'.\n", options.config_file ); } if ( ReadRobinhoodConfig( MODULE_MASK_FS_SCAN | MODULE_MASK_ENTRY_PROCESSOR, options.config_file, err_msg, &rh_config, FALSE ) ) { fprintf( stderr, "Error reading configuration file '%s': %s\n", options.config_file, err_msg ); exit( 1 ); } process_config_file = options.config_file; /* set global configuration */ global_config = rh_config.global_config; /* set policies info */ policies = rh_config.policies; if (options.force_log_level) rh_config.log_config.debug_level = options.log_level; else rh_config.log_config.debug_level = LVL_CRIT; /* no event message */ /* Set logging to stderr */ strcpy( rh_config.log_config.log_file, "stderr" ); strcpy( rh_config.log_config.report_file, "stderr" ); strcpy( rh_config.log_config.alert_file, "stderr" ); /* Initialize logging */ rc = InitializeLogs( bin, &rh_config.log_config ); if ( rc ) { fprintf( stderr, "Error opening log files: rc=%d, errno=%d: %s\n", rc, errno, strerror( errno ) ); exit( rc ); } /* Initialize filesystem access */ rc = InitFS(); if (rc) exit(rc); #ifdef _HSM_LITE rc = Backend_Start( &rh_config.backend_config, options.flags ); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error initializing backend" ); exit( 1 ); } #endif /* Initialize list manager */ rc = ListMgr_Init( &rh_config.lmgr_config, FALSE ); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error %d initializing list manager", rc ); exit( rc ); } else DisplayLog( LVL_VERB, DIFF_TAG, "ListManager successfully initialized" ); if ( CheckLastFS( ) != 0 ) exit( 1 ); if (options.diff_mask) rh_config.entry_proc_config.diff_mask = options.diff_mask; else { /* parse "all" */ char tmpstr[] = "all"; if (parse_diff_mask(tmpstr, &rh_config.entry_proc_config.diff_mask, err_msg)) { DisplayLog(LVL_CRIT, DIFF_TAG, "unexpected error parsing diff mask: %s", err_msg); exit(1); } } #ifdef LUSTRE_DUMP_FILES if (options.diff_arg.apply == APPLY_FS && !(options.flags & FLAG_DRY_RUN)) { /* open the file to write LOV EA and FID remapping */ if (!EMPTY_STRING(options.output_dir)) { char fname[RBH_PATH_MAX]; if (mkdir(options.output_dir, 0700) && (errno != EEXIST)) { DisplayLog(LVL_CRIT, DIFF_TAG, "Failed to create directory %s: %s", options.output_dir, strerror(errno)); exit(1); } snprintf(fname, RBH_PATH_MAX-1, "%s/"LOVEA_FNAME, options.output_dir); options.diff_arg.lovea_file = fopen(fname, "w"); if (options.diff_arg.lovea_file == NULL) { DisplayLog(LVL_CRIT, DIFF_TAG, "Failed to open %s for writting: %s", fname, strerror(errno)); exit(1); } snprintf(fname, RBH_PATH_MAX-1, "%s/"FIDREMAP_FNAME, options.output_dir); options.diff_arg.fid_remap_file = fopen(fname, "w"); if (options.diff_arg.fid_remap_file == NULL) { DisplayLog(LVL_CRIT, DIFF_TAG, "Failed to open %s for writting: %s", fname, strerror(errno)); exit(1); } } } #endif /* if no DB apply action is specified, can't use md_update field for checking * removed entries. So, create a special tag for that. */ if ((options.diff_arg.apply != APPLY_DB) || (options.flags & FLAG_DRY_RUN)) { fprintf(stderr, "Preparing diff table...\n"); /* create a connexion to the DB. this is safe to use the global lmgr var * as statistics thread is not running */ if (!ensure_db_access()) exit(1); /* create a tag to clear entries after the scan */ /* There could be several diff running in parallel, * so set a suffix to avoid conflicts */ sprintf(tag_name, "DIFF_%u", (unsigned int) getpid()); options.diff_arg.db_tag = tag_name; /* add filter for partial scan */ if (options.partial_scan) { lmgr_filter_t filter; filter_value_t val; lmgr_simple_filter_init( &filter ); char tmp[RBH_PATH_MAX]; strcpy(tmp, options.partial_scan_path); strcat(tmp, "/*"); val.value.val_str = tmp; lmgr_simple_filter_add(&filter, ATTR_INDEX_fullpath, LIKE, val, 0); rc = ListMgr_CreateTag(&lmgr, tag_name, &filter, FALSE); lmgr_simple_filter_free(&filter); } else rc = ListMgr_CreateTag(&lmgr, tag_name, NULL, FALSE); if (rc) exit(rc); } /* Initialise Pipeline */ rc = EntryProcessor_Init(&rh_config.entry_proc_config, DIFF_PIPELINE, options.flags, &options.diff_arg); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error %d initializing EntryProcessor pipeline", rc ); goto clean_tag; } else DisplayLog( LVL_VERB, DIFF_TAG, "EntryProcessor successfully initialized" ); fprintf(stderr, "Starting scan\n"); /* print header to indicate the content of diff * #<diff cmd> * ---fs[=/subdir] * +++db */ for (i = 0; i < argc; i++) printf("%s%s", i==0?"# ":" ", argv[i]); printf("\n"); if (options.diff_arg.apply == APPLY_FS) { if (options.partial_scan) printf("---fs=%s\n",options.partial_scan_path); else printf("---fs\n"); printf("+++db\n"); } else { printf("---db\n"); if (options.partial_scan) printf("+++fs=%s\n",options.partial_scan_path); else printf("+++fs\n"); } /* Start FS scan */ if (options.partial_scan) rc = FSScan_Start(&rh_config.fs_scan_config, options.flags, options.partial_scan_path); else rc = FSScan_Start(&rh_config.fs_scan_config, options.flags, NULL); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error %d initializing FS Scan module", rc ); goto clean_tag; } else DisplayLog( LVL_VERB, DIFF_TAG, "FS Scan module successfully initialized" ); /* Flush logs now, to have a trace in the logs */ FlushLogs( ); /* both pipeline and scan are now running, can now trap events and display stats */ /* create signal handling thread */ rc = pthread_create( &sig_thr, NULL, signal_handler_thr, NULL ); if ( rc ) { DisplayLog( LVL_CRIT, DIFF_TAG, "Error starting signal handler thread: %s", strerror( errno ) ); goto clean_tag; } else DisplayLog( LVL_VERB, DIFF_TAG, "Signal handler thread started successfully" ); pthread_create(&stat_thread, NULL, stats_thr, NULL); /* wait for FS scan to end */ FSScan_Wait( ); DisplayLog( LVL_MAJOR, DIFF_TAG, "FS Scan finished" ); /* Pipeline must be flushed */ EntryProcessor_Terminate( TRUE ); #ifdef LUSTRE_DUMP_FILES /* flush the lovea file */ if (options.diff_arg.lovea_file) { fprintf(stderr, " > LOV EA information written to %s/"LOVEA_FNAME"\n", options.output_dir); fclose(options.diff_arg.lovea_file); } if (options.diff_arg.fid_remap_file) { fprintf(stderr, " > FID remapping written to %s/"FIDREMAP_FNAME"\n", options.output_dir); fclose(options.diff_arg.fid_remap_file); } #endif fprintf(stderr, "End of scan\n"); DisplayLog( LVL_MAJOR, DIFF_TAG, "All tasks done! Exiting." ); rc = 0; clean_tag: /* destroy the tag before exit */ if (options.diff_arg.db_tag != NULL && ensure_db_access()) { fprintf(stderr, "Cleaning diff table...\n"); ListMgr_DestroyTag(&lmgr, options.diff_arg.db_tag); } exit(rc); return rc; /* for compiler */ }
static void *signal_handler_thr( void *arg ) { struct sigaction act_sigterm; struct sigaction act_sigusr; /* create signal handlers */ memset( &act_sigterm, 0, sizeof( act_sigterm ) ); act_sigterm.sa_flags = 0; act_sigterm.sa_handler = terminate_handler; if ( sigaction( SIGTERM, &act_sigterm, NULL ) == -1 || sigaction( SIGINT, &act_sigterm, NULL ) == -1 ) { DisplayLog( LVL_CRIT, SIGHDL_TAG, "Error while setting signal handlers for SIGTERM and SIGINT: %s", strerror( errno ) ); if (options.diff_arg.db_tag != NULL && ensure_db_access()) { fprintf(stderr, "Cleaning diff table...\n"); ListMgr_DestroyTag(&lmgr, options.diff_arg.db_tag); } exit( 1 ); } else DisplayLog( LVL_EVENT, SIGHDL_TAG, "Signals SIGTERM and SIGINT (daemon shutdown) are ready to be used" ); memset( &act_sigusr, 0, sizeof( act_sigusr ) ); act_sigusr.sa_flags = 0; act_sigusr.sa_handler = usr_handler; if ( sigaction( SIGUSR1, &act_sigusr, NULL ) == -1 ) { DisplayLog( LVL_CRIT, SIGHDL_TAG, "Error while setting signal handlers for SIGUSR1: %s", strerror( errno ) ); if (options.diff_arg.db_tag != NULL && ensure_db_access()) { fprintf(stderr, "Cleaning diff table...\n"); ListMgr_DestroyTag(&lmgr, options.diff_arg.db_tag); /* make sure written data is flushed */ if (options.diff_arg.lovea_file) fflush(options.diff_arg.lovea_file); if (options.diff_arg.fid_remap_file) fflush(options.diff_arg.fid_remap_file); } exit( 1 ); } else DisplayLog( LVL_EVENT, SIGHDL_TAG, "Signal SIGUSR1 (stats dump) is ready to be used" ); /* signal flag checking loop */ while ( 1 ) { /* check for signal every second */ rh_sleep( 1 ); if ( terminate_sig != 0 ) { if ( terminate_sig == SIGTERM ) DisplayLog( LVL_MAJOR, SIGHDL_TAG, "SIGTERM received: performing clean daemon shutdown" ); else if ( terminate_sig == SIGINT ) DisplayLog( LVL_MAJOR, SIGHDL_TAG, "SIGINT received: performing clean daemon shutdown" ); FlushLogs( ); /* stop FS scan (blocking) */ FSScan_Terminate( ); FlushLogs( ); /* drop pipeline waiting operations and terminate threads */ EntryProcessor_Terminate( FALSE ); FlushLogs( ); #ifdef _HSM_LITE /* shutdown backend access */ Backend_Stop(); #endif DisplayLog( LVL_MAJOR, SIGHDL_TAG, "Exiting." ); FlushLogs( ); if (options.diff_arg.db_tag != NULL && ensure_db_access()) { fprintf(stderr, "Cleaning diff table...\n"); ListMgr_DestroyTag(&lmgr, options.diff_arg.db_tag); /* make sure written data is flushed */ if (options.diff_arg.lovea_file) fflush(options.diff_arg.lovea_file); if (options.diff_arg.fid_remap_file) fflush(options.diff_arg.fid_remap_file); } /* indicate the process terminated due to a signal */ exit( 128 + terminate_sig ); } else if ( dump_sig ) { DisplayLog( LVL_MAJOR, SIGHDL_TAG, "SIGUSR1 received: dumping stats" ); if (!ensure_db_access()) return NULL; dump_stats(&lmgr); dump_sig = FALSE; } } }
Bool HawkLogServer::Run() { if (!m_pLogFile && !m_pLogDB) return false; if (m_bRunning || !m_pRecvBuf || !m_pLogCache) return false; m_bRunning = true; UInt32 iFlushTime = HawkOSOperator::GetTickCount(); while(m_bRunning) { UInt32 iCurTime = HawkOSOperator::GetTickCount(); if (!m_sSocket.UpdateEvent(HEVENT_READ)) { if (iCurTime - iFlushTime >= (UInt32)m_iCacheTime) { FlushLogs(); iFlushTime = iCurTime; } else { HawkSleep(DEFAULT_SLEEP); } continue; } m_pRecvBuf->Clear(); Size_t iRecv = (Size_t)m_pRecvBuf->Capacity(); SocketAddr sAddr; if(!m_sSocket.ReceiveFrom(m_pRecvBuf->Begin(), iRecv, sAddr)) continue; m_pRecvBuf->Resize((UInt32)iRecv); Protocol* pProto = 0; try { pProto = P_ProtocolManager->Decode(*m_pRecvBuf); if (!pProto) continue; } catch (HawkException& rhsExcep) { #ifdef _DEBUG HawkFmtPrint("Exception: %s", rhsExcep.GetMsg().c_str()); #endif continue; } ProtoType eType = pProto->GetType(); if(eType == SysProtocol::SYS_LOG_MSG) { SysProtocol::Sys_LogMsg* pCmd = (SysProtocol::Sys_LogMsg*)pProto; CacheLogs(pCmd->m_iLogId, pCmd->m_iType, pCmd->m_sKey, pCmd->m_sMsg); } P_ProtocolManager->ReleaseProto(pProto); } FlushLogs(); return true; }
Bool HawkLogServer::CacheLogs(Int32 iLogId, Int32 iType, const UString& sKey, const UString& sMsg) { memset(m_pFmtBuf, 0, PAGE_SIZE); if (!sKey.size() || !sMsg.size()) return false; if (m_pLogFile) { //格式化日志信息 sprintf((Char*)m_pFmtBuf, "LogId: %d, Type: %d, Time: %s, Key: %s, Msg: %s", iLogId, iType, HawkOSOperator::GetTimeString().c_str(), sKey.c_str(), sMsg.c_str()); UInt32 iSize = (UInt32)strlen((Char*)m_pFmtBuf); if (iSize + m_pLogCache->Size() >= m_pLogCache->Capacity() - 2) { //日志落地 FlushLogs(); } //屏幕输出 if (m_bConsole) HawkPrint(HawkStringUtil::ToString(m_pFmtBuf)); //写入缓存 m_pLogCache->Append(m_pFmtBuf, iSize); return true; } else if (m_pLogDB) { //格式化日志信息 if(m_pLogCache->Size()) { sprintf((Char*)m_pFmtBuf, ",(%d, %d, '%s', '%s', '%s')", iLogId, iType, HawkOSOperator::GetTimeString().c_str(), sKey.c_str(), sMsg.c_str()); } else { sprintf((Char*)m_pFmtBuf, "INSERT INTO logs(LogId, LogType, LogTime, LogKey, LogMsg) VALUES(%d, %d, '%s', '%s', '%s')", iLogId, iType, HawkOSOperator::GetTimeString().c_str(), sKey.c_str(), sMsg.c_str()); } UInt32 iSize = (UInt32)strlen((Char*)m_pFmtBuf); if (iSize + m_pLogCache->Size() >= m_pLogCache->Capacity() - 2) { //日志落地 FlushLogs(); //重新Format日志格式 sprintf((Char*)m_pFmtBuf, "INSERT INTO logs(LogId, LogType, LogTime, LogKey, LogMsg) VALUES(%d, %d, '%s', '%s', '%s')", iLogId, iType, HawkOSOperator::GetTimeString().c_str(), sKey.c_str(), sMsg.c_str()); iSize = (UInt32)strlen((Char*)m_pFmtBuf); } //屏幕输出 if (m_bConsole) HawkPrint(HawkStringUtil::ToString(m_pFmtBuf)); //写入缓存 m_pLogCache->Append(m_pFmtBuf, iSize); return true; } return false; }