int EntryProc_db_apply( struct entry_proc_op_t *p_op, lmgr_t * p_lmgr ) { int rc; const pipeline_stage_t *stage_info = &entry_proc_pipeline[p_op->pipeline_stage]; #ifdef DEBUG printf( "stage %s - record #%u - id=[%llu,%u] - thread %lx\n", stage_info->stage_name, ( unsigned int ) p_op->entry_id.f_ver, p_op->entry_id.f_seq, p_op->entry_id.f_oid, pthread_self( ) ); #endif #ifdef _HAVE_FID if ( p_op->entry_id.f_ver % ( NB_OP_ENQUEUE / 10 ) == 0 ) EntryProcessor_DumpCurrentStages( ); #endif /* simulate slow DB query */ usleep( 1000 ); #ifdef _HAVE_FID if ( ( unsigned int ) p_op->entry_id.f_ver == NB_OP_ENQUEUE - 1 ) exit( 0 ); #endif /* acknowledge */ rc = EntryProcessor_Acknowledge( p_op, 0, TRUE ); if ( rc ) printf( "Error acknowledging stage %s\n", stage_info->stage_name ); return 0; }
int main( int argc, char **argv ) { int i, rc; pthread_t enqueuer; void *dummy; pthread_attr_t attr; robinhood_config_t config; char err_msg[2048]; srand( time( NULL ) + getpid( ) ); if ( argc < 2 ) { fprintf( stderr, "Usage : test_entry_proc <config_file>\n" ); exit( 1 ); } /* only parse config for mandatory module */ if ( ReadRobinhoodConfig( MODULE_MASK_ENTRY_PROCESSOR, argv[1], err_msg, &config ) ) { fprintf( stderr, "Error reading configuration:\n%s\n", err_msg ); exit( 1 ); } InitializeLogs( "test_entry_proc", &config.log_config ); rc = ListMgr_Init( &config.lmgr_config ); if ( rc ) exit( rc ); /* this starts work threads */ if ( EntryProcessor_Init( &config.entry_proc_config ) ) { printf( "init error" ); return -1; } pthread_attr_init( &attr ); pthread_attr_setstacksize( &attr, 32 * 4096 ); pthread_attr_setscope( &attr, PTHREAD_SCOPE_SYSTEM ); /* create enqueue thread */ pthread_create( &enqueuer, &attr, enqueue_thread, NULL ); while ( 1 ) /* exit is done by db_apply thread */ { sleep( 2 ); EntryProcessor_DumpCurrentStages( ); } /* wait for enqueuer thread */ pthread_join( enqueuer, &dummy ); return 0; }
int EntryProc_db_apply( struct entry_proc_op_t *p_op, lmgr_t * lmgr ) { int rc; const pipeline_stage_t *stage_info = &entry_proc_pipeline[p_op->pipeline_stage]; #ifdef DEBUG printf( "stage %s - record #%u - id=[%llu,%u] - thread %lx\n", stage_info->stage_name, ( unsigned int ) p_op->entry_id.f_ver, p_op->entry_id.f_seq, p_op->entry_id.f_oid, pthread_self( ) ); #endif #ifdef _HAVE_FID if ( p_op->entry_id.f_ver % ( NB_OP_ENQUEUE / 10 ) == 0 ) EntryProcessor_DumpCurrentStages( ); #endif /* insert to DB */ switch ( p_op->db_op_type ) { case OP_TYPE_INSERT: rc = ListMgr_Insert( lmgr, &p_op->entry_id, &p_op->entry_attr ); break; case OP_TYPE_UPDATE: rc = ListMgr_Update( lmgr, &p_op->entry_id, &p_op->entry_attr ); break; default: printf( "unhandled\n" ); rc = -1; } if ( rc ) printf( "ERROR: ListMgr operation returned %d\n", rc ); #ifdef _HAVE_FID if ( ( unsigned int ) p_op->entry_id.f_ver == NB_OP_ENQUEUE - 1 ) exit( 0 ); #endif if ( p_op->callback_func ) p_op->callback_func( lmgr, p_op, p_op->callback_param ); /* acknoledge */ rc = EntryProcessor_Acknowledge( p_op, 0, TRUE ); if ( rc ) printf( "Error acknowledging stage %s\n", stage_info->stage_name ); return 0; }
int EntryProc_get_attr( struct entry_proc_op_t *p_op, lmgr_t * p_lmgr ) #endif { int rc; entry_id_t id; const pipeline_stage_t *stage_info = &entry_proc_pipeline[p_op->pipeline_stage]; /* set id */ #ifdef _HAVE_FID id.f_seq = myrand( 1000 ); id.f_oid = myrand( 1000 ); id.f_ver = p_op->extra_info.log_record.record_id; #else id.device = myrand( 1000 ); id.inode = myrand( 1000 ); #endif EntryProcessor_SetEntryId( p_op, &id ); #ifdef DEBUG printf( "stage %s - record #%u - id=[%llu,%u] - thread %lx\n", stage_info->stage_name, ( unsigned int ) p_op->entry_id.f_ver, p_op->entry_id.f_seq, p_op->entry_id.f_oid, pthread_self( ) ); EntryProcessor_DumpCurrentStages( ); #endif /* acknowledge */ rc = EntryProcessor_Acknowledge( p_op, p_op->pipeline_stage + 1, FALSE ); if ( rc ) printf( "Error acknowledging stage %s\n", stage_info->stage_name ); /* if processing is async, simulate a long operation */ if ( stage_info->stage_flags & STAGE_FLAG_ASYNC ) { /* do this for 1 entry out of 1000 */ if ( myrand( 1000 ) == 0 ) { usleep( 10000 ); DisplayLog( LVL_DEBUG, "CheckExist", "End of async processing for stage %s", stage_info->stage_name ); } } return 0; }
static void dump_stats( lmgr_t * lmgr ) { char tmp_buff[256]; time_t now; struct tm date; now = time( NULL ); strftime( tmp_buff, 256, "%Y/%m/%d %T", localtime_r( &now, &date ) ); DisplayLog( LVL_MAJOR, "STATS", "==================== Dumping stats at %s =====================", tmp_buff ); DisplayLog( LVL_MAJOR, "STATS", "Diff start time: %s", start_time_str ); FSScan_DumpStats(); EntryProcessor_DumpCurrentStages(); /* Flush stats */ FlushLogs( ); }
int EntryProc_check_exist( struct entry_proc_op_t *p_op, lmgr_t * p_lmgr ) { int rc; const pipeline_stage_t *stage_info = &entry_proc_pipeline[p_op->pipeline_stage]; #ifdef DEBUG printf( "stage %s - record #%u - id=[%llu,%u] - thread %lx\n", stage_info->stage_name, ( unsigned int ) p_op->entry_id.f_ver, p_op->entry_id.f_seq, p_op->entry_id.f_oid, pthread_self( ) ); EntryProcessor_DumpCurrentStages( ); #endif /* acknowledge */ /* and randomly return 1 entry out of 100 */ if ( myrand( 100 ) == 0 ) { rc = EntryProcessor_Acknowledge( p_op, p_op->pipeline_stage + 1, TRUE ); if ( rc ) printf( "Error acknowledging stage %s\n", stage_info->stage_name ); return rc; } else { rc = EntryProcessor_Acknowledge( p_op, p_op->pipeline_stage + 1, FALSE ); if ( rc ) printf( "Error acknowledging stage %s\n", stage_info->stage_name ); } /* if processing is async, simulate a long operation */ if ( stage_info->stage_flags & STAGE_FLAG_ASYNC ) { /* do this for 1 entry out of 1000 */ if ( myrand( 1000 ) == 0 ) { usleep( 100000 ); DisplayLog( LVL_DEBUG, "CheckExist", "End of async processing for stage %s", stage_info->stage_name ); } } return 0; }
int main( int argc, char **argv ) { int i; pthread_t worker[NB_WORKER]; void *dummy; pthread_attr_t attr; robinhood_config_t config; char err_msg[2048]; srand( time( NULL ) + getpid( ) ); if ( argc < 2 ) { fprintf( stderr, "Usage : test_fs_scan <config_file>\n" ); exit( 1 ); } /* Initilize uidgid cache */ if ( InitUidGid_Cache( ) ) { fprintf( stderr, "Error initializing uid/gid cache\n" ); exit( 1 ); } /* only parse config for mandatory module */ if ( ReadRobinhoodConfig ( MODULE_MASK_ENTRY_PROCESSOR | MODULE_MASK_INFO_COLLECTOR, argv[1], err_msg, &config ) ) { fprintf( stderr, "Error reading configuration:\n%s\n", err_msg ); exit( 1 ); } /* set global configuration */ global_config = config.global_config; /* set policies */ purge_policy = config.purge_policy; InitializeLogs( "test_fs_scan", &config.log_config ); /* Initialise List Mgr */ if ( ListMgr_Init( &config.lmgr_config ) ) { printf( "List Mgr init error" ); return -1; } /* Initialise Pipeline */ if ( EntryProcessor_Init( &config.entry_proc_config ) ) { printf( "Pipeline init error" ); return -1; } /* Start info collectors */ if ( Start_Info_Collectors( &config.info_collect_config ) ) { printf( "Info collector init error" ); return -1; } pthread_attr_init( &attr ); pthread_attr_setstacksize( &attr, 32 * 4096 ); pthread_attr_setscope( &attr, PTHREAD_SCOPE_SYSTEM ); while ( 1 ) /* exit is done by db_apply thread */ { sleep( 30 ); EntryProcessor_DumpCurrentStages( ); Dump_Info_Collectors_Stats( ); } for ( i = 0; i < NB_WORKER; i++ ) pthread_join( worker[i], &dummy ); return 0; }