void test_CreateAndDestroy(void)
{
HObs hObs = NULL;
hObs = obsLib_new( 0, 0 );
CU_ASSERT_TRUE_FATAL( rpal_memory_isValid( hObs ) );
obsLib_free( hObs );
}
void test_addPattern(void)
{
HObs hObs = NULL;
RU8 pattern[] = { 0x01, 0x02, 0x03, 0x04 };
RU32 context = 0;
hObs = obsLib_new( 0, 0 );
CU_ASSERT_TRUE_FATAL( rpal_memory_isValid( hObs ) );
CU_ASSERT_TRUE_FATAL( obsLib_addPattern( hObs, (RPU8)&pattern, sizeof( pattern ), &context ) );
obsLib_free( hObs );
}
void test_singlePattern(void)
{
HObs hObs = NULL;
RU8 pattern[] = { 0x01, 0x02, 0x03, 0x04 };
RU8 buffer1[] = { 0x02, 0x04, 0xFF, 0xEF, 0x01, 0x02, 0x03, 0x04 };
RU8 buffer2[] = { 0x02, 0x04, 0xFF, 0xEF, 0x01, 0x02, 0x03, 0x04, 0xEE, 0x6F };
RU8 buffer3[] = { 0x02, 0x04, 0xFF, 0xEF, 0x01, 0x02, 0x01, 0x04, 0xEE, 0x6F };
RU8 buffer4[] = { 0x02, 0x04, 0xFF, 0xEF, 0x01, 0x02, 0x03, 0x04, 0xEE, 0x6F, 0x01, 0x02, 0x03, 0x04 };
RU32 context = 0;
PVOID hitCtx = NULL;
RU8* hitLoc = NULL;
hObs = obsLib_new( 0, 0 );
CU_ASSERT_TRUE_FATAL( rpal_memory_isValid( hObs ) );
CU_ASSERT_TRUE_FATAL( obsLib_addPattern( hObs, (RPU8)&pattern, sizeof( pattern ), &context ) );
// 1 pattern found end of buffer
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer1, sizeof( buffer1 ) ) );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context );
CU_ASSERT_EQUAL( hitLoc, buffer1 + sizeof( buffer1 ) - 4 );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
// 1 pattern found middle of buffer
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer2, sizeof( buffer2 ) ) );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context );
CU_ASSERT_EQUAL( hitLoc, buffer2 + sizeof( buffer2 ) - 6 );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
// 0 pattern found
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer3, sizeof( buffer3 ) ) );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
// 2 pattern found end and middle of buffer
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer4, sizeof( buffer4 ) ) );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context );
CU_ASSERT_EQUAL( hitLoc, buffer4 + sizeof( buffer4 ) - 10 );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context );
CU_ASSERT_EQUAL( hitLoc, buffer4 + sizeof( buffer4 ) - 4 );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
obsLib_free( hObs );
}
RBOOL
collector_18_init
(
HbsState* hbsState,
rSequence config
)
{
RBOOL isSuccess = FALSE;
rList extensions = NULL;
rList patterns = NULL;
RPCHAR strA = NULL;
RPCHAR tmpA = NULL;
RPWCHAR strW = NULL;
RPWCHAR tmpW = NULL;
RU32 maxSize = 0;
RBOOL isCaseInsensitive = FALSE;
if( NULL != hbsState )
{
#ifdef RPAL_PLATFORM_WINDOWS
// On Windows files and paths are not case sensitive.
isCaseInsensitive = TRUE;
#endif
if( NULL == config ||
rSequence_getLIST( config, RP_TAGS_EXTENSIONS, &extensions ) ||
rSequence_getLIST( config, RP_TAGS_PATTERNS, &patterns ) )
{
if( NULL != ( cacheMutex = rMutex_create() ) &&
NULL != ( matcherA = obsLib_new( 0, 0 ) ) &&
NULL != ( matcherW = obsLib_new( 0, 0 ) ) )
{
cacheSize = 0;
if( NULL != config &&
rSequence_getRU32( config, RP_TAGS_MAX_SIZE, &maxSize ) )
{
cacheMaxSize = maxSize;
}
else
{
cacheMaxSize = MAX_CACHE_SIZE;
}
if( NULL != ( documentCache = HbsRingBuffer_new( 0, cacheMaxSize ) ) )
{
if( NULL == config )
{
// As a default we'll cache all new files
obsLib_addPattern( matcherA, (RPU8)"", sizeof( RCHAR ), NULL );
obsLib_addPattern( matcherW, (RPU8)_WCH(""), sizeof( RWCHAR ), NULL );
}
else
{
// If a config was provided we'll cache only certain extensions
// specified.
while( rList_getSTRINGA( extensions, RP_TAGS_EXTENSION, &strA ) )
{
if( rpal_string_expand( strA, &tmpA ) )
{
obsLib_addStringPatternA( matcherA, tmpA, TRUE, isCaseInsensitive, NULL );
rpal_memory_free( tmpA );
}
if( NULL != ( strW = rpal_string_atow( strA ) ) )
{
if( rpal_string_expandw( strW, &tmpW ) )
{
obsLib_addStringPatternW( matcherW, tmpW, TRUE, isCaseInsensitive, NULL );
rpal_memory_free( tmpW );
}
rpal_memory_free( strW );
}
}
while( rList_getSTRINGW( extensions, RP_TAGS_EXTENSION, &strW ) )
{
if( rpal_string_expandw( strW, &tmpW ) )
{
obsLib_addStringPatternW( matcherW, tmpW, TRUE, isCaseInsensitive, NULL );
rpal_memory_free( tmpW );
}
if( NULL != ( strA = rpal_string_wtoa( strW ) ) )
{
if( rpal_string_expand( strA, &tmpA ) )
{
obsLib_addStringPatternA( matcherA, tmpA, TRUE, isCaseInsensitive, NULL );
rpal_memory_free( tmpA );
}
rpal_memory_free( strA );
}
}
while( rList_getSTRINGA( patterns, RP_TAGS_STRING_PATTERN, &strA ) )
{
if( rpal_string_expand( strA, &tmpA ) )
{
obsLib_addStringPatternA( matcherA, tmpA, FALSE, isCaseInsensitive, NULL );
rpal_memory_free( tmpA );
}
if( NULL != ( strW = rpal_string_atow( strA ) ) )
{
if( rpal_string_expandw( strW, &tmpW ) )
{
obsLib_addStringPatternW( matcherW, tmpW, FALSE, isCaseInsensitive, NULL );
rpal_memory_free( tmpW );
}
rpal_memory_free( strW );
}
}
while( rList_getSTRINGW( patterns, RP_TAGS_STRING_PATTERN, &strW ) )
{
if( rpal_string_expandw( strW, &tmpW ) )
{
obsLib_addStringPatternW( matcherW, tmpW, FALSE, isCaseInsensitive, NULL );
rpal_memory_free( tmpW );
}
if( NULL != ( strA = rpal_string_wtoa( strW ) ) )
{
if( rpal_string_expand( strA, &tmpA ) )
{
obsLib_addStringPatternA( matcherA, tmpA, FALSE, isCaseInsensitive, NULL );
rpal_memory_free( tmpA );
}
rpal_memory_free( strA );
}
}
}
if( rQueue_create( &createQueue, _freeEvt, 200 ) &&
notifications_subscribe( RP_TAGS_NOTIFICATION_FILE_CREATE, NULL, 0, createQueue, NULL ) &&
notifications_subscribe( RP_TAGS_NOTIFICATION_GET_DOCUMENT_REQ, NULL, 0, NULL, getDocument ) &&
rThreadPool_task( hbsState->hThreadPool, parseDocuments, NULL ) )
{
isSuccess = TRUE;
}
}
}
if( !isSuccess )
{
notifications_unsubscribe( RP_TAGS_NOTIFICATION_FILE_CREATE, createQueue, NULL );
notifications_unsubscribe( RP_TAGS_NOTIFICATION_GET_DOCUMENT_REQ, NULL, getDocument );
rQueue_free( createQueue );
createQueue = NULL;
obsLib_free( matcherA );
obsLib_free( matcherW );
HbsRingBuffer_free( documentCache );
matcherA = NULL;
matcherW = NULL;
documentCache = NULL;
rMutex_free( cacheMutex );
cacheMutex = NULL;
}
}
}
return isSuccess;
}
RBOOL
collector_22_init
(
HbsState* hbsState,
rSequence config
)
{
RBOOL isSuccess = FALSE;
rList patterns = NULL;
rSequence pattern = NULL;
RPCHAR strA = NULL;
RPWCHAR strW = NULL;
RPNCHAR tmpN = NULL;
RU8 patternId = 0;
RU32 i = 0;
if( NULL != hbsState &&
NULL != ( g_extensions = obsLib_new( 0, 0 ) ) )
{
if( rSequence_getLIST( config, RP_TAGS_PATTERNS, &patterns ) )
{
while( rList_getSEQUENCE( patterns, RP_TAGS_RULE, &pattern ) )
{
if( rSequence_getRU8( pattern, RP_TAGS_RULE_NAME, &patternId ) )
{
if( 64 < patternId || 0 == patternId )
{
rpal_debug_critical( "rule id must be below 64 and 1-based." );
continue;
}
// Base the pattern id to 0
patternId--;
if( rSequence_getSTRINGA( pattern, RP_TAGS_EXTENSION, &strA ) &&
NULL != ( tmpN = rpal_string_aton( strA ) ) )
{
_addPattern( g_extensions, tmpN, TRUE, NUMBER_TO_PTR( patternId ) );
rpal_memory_free( tmpN );
}
if( rSequence_getSTRINGW( pattern, RP_TAGS_EXTENSION, &strW ) &&
NULL != ( tmpN = rpal_string_wton( strW ) ) )
{
_addPattern( g_extensions, tmpN, TRUE, NUMBER_TO_PTR( patternId ) );
rpal_memory_free( tmpN );
}
if( rSequence_getSTRINGA( pattern, RP_TAGS_STRING_PATTERN, &strA ) &&
NULL != ( tmpN = rpal_string_aton( strA ) ) )
{
_addPattern( g_extensions, tmpN, FALSE, NUMBER_TO_PTR( patternId ) );
rpal_memory_free( tmpN );
}
if( rSequence_getSTRINGW( pattern, RP_TAGS_STRING_PATTERN, &strW ) &&
NULL != ( tmpN = rpal_string_wton( strW ) ) )
{
_addPattern( g_extensions, tmpN, FALSE, NUMBER_TO_PTR( patternId ) );
rpal_memory_free( tmpN );
}
}
}
if( NULL != ( g_mutex = rMutex_create() ) &&
NULL != ( g_procContexts = rpal_vector_new() ) &&
notifications_subscribe( RP_TAGS_NOTIFICATION_FILE_CREATE, NULL, 0, NULL, processFileIo ) &&
notifications_subscribe( RP_TAGS_NOTIFICATION_FILE_DELETE, NULL, 0, NULL, processFileIo ) &&
notifications_subscribe( RP_TAGS_NOTIFICATION_FILE_MODIFIED, NULL, 0, NULL, processFileIo ) &&
notifications_subscribe( RP_TAGS_NOTIFICATION_FILE_READ, NULL, 0, NULL, processFileIo ) &&
notifications_subscribe( RP_TAGS_NOTIFICATION_NEW_PROCESS, NULL, 0, NULL, processNewProcesses ) &&
notifications_subscribe( RP_TAGS_NOTIFICATION_EXISTING_PROCESS, NULL, 0, NULL, processNewProcesses ) &&
notifications_subscribe( RP_TAGS_NOTIFICATION_TERMINATE_PROCESS, NULL, 0, NULL, processTerminateProcesses ) )
{
isSuccess = TRUE;
}
}
}
if( !isSuccess )
{
notifications_unsubscribe( RP_TAGS_NOTIFICATION_FILE_CREATE, NULL, processFileIo );
notifications_unsubscribe( RP_TAGS_NOTIFICATION_FILE_DELETE, NULL, processFileIo );
notifications_unsubscribe( RP_TAGS_NOTIFICATION_FILE_MODIFIED, NULL, processFileIo );
notifications_unsubscribe( RP_TAGS_NOTIFICATION_FILE_READ, NULL, processFileIo );
notifications_unsubscribe( RP_TAGS_NOTIFICATION_NEW_PROCESS, NULL, processNewProcesses );
notifications_unsubscribe( RP_TAGS_NOTIFICATION_EXISTING_PROCESS, NULL, processNewProcesses );
notifications_unsubscribe( RP_TAGS_NOTIFICATION_TERMINATE_PROCESS, NULL, processTerminateProcesses );
obsLib_free( g_extensions );
g_extensions = NULL;
if( NULL != g_procContexts )
{
for( i = 0; i < g_procContexts->nElements; i++ )
{
rpal_memory_free( ( (ProcExtInfo*)g_procContexts->elements[ i ] )->processPath );
rpal_memory_free( g_procContexts->elements[ i ] );
}
}
rpal_vector_free( g_procContexts );
g_procContexts = NULL;
rMutex_free( g_mutex );
g_mutex = NULL;
}
return isSuccess;
}
static
HObs
_getModuleDiskStringSample
(
RPNCHAR modulePath,
RU32* pLastScratch,
rEvent isTimeToStop,
LibOsPerformanceProfile* perfProfile
)
{
HObs sample = NULL;
RPU8 scratch = NULL;
rFile hFile = NULL;
RU32 read = 0;
RPU8 start = NULL;
RPU8 end = NULL;
RU32 toSkip = 0;
RU32 longestLength = 0;
RBOOL isUnicode = FALSE;
RPU8 sampleNumber = 0;
rBloom stringsSeen = NULL;
RU64 readOffset = 0;
UNREFERENCED_PARAMETER( isTimeToStop );
if( NULL != modulePath &&
NULL != pLastScratch )
{
readOffset = *pLastScratch * _SCRATCH_SIZE;
if( NULL != ( stringsSeen = rpal_bloom_create( _MAX_DISK_SAMPLE_SIZE, 0.0001 ) ) )
{
if( NULL != ( scratch = rpal_memory_alloc( _SCRATCH_SIZE ) ) )
{
if( rFile_open( modulePath, &hFile, RPAL_FILE_OPEN_EXISTING |
RPAL_FILE_OPEN_READ ) )
{
if( readOffset == rFile_seek( hFile,
readOffset,
rFileSeek_SET ) &&
0 != ( read = rFile_readUpTo( hFile, _SCRATCH_SIZE, scratch ) ) )
{
if( NULL != ( sample = obsLib_new( _MAX_DISK_SAMPLE_SIZE, 0 ) ) )
{
( *pLastScratch )++;
start = scratch;
end = scratch + read;
// We parse for strings up to 'read', we don't care about the
// memory boundary, we might truncate some strings but we're
// sampling anyway.
while( !rEvent_wait( isTimeToStop, 0 ) &&
( start >= scratch ) && ( start >= scratch ) &&
( start + _MIN_SAMPLE_STR_LEN ) < ( scratch + read ) &&
_MAX_DISK_SAMPLE_SIZE >= PTR_TO_NUMBER( sampleNumber ) )
{
libOs_timeoutWithProfile( perfProfile, TRUE, isTimeToStop );
isUnicode = FALSE;
if( _longestString( (RPCHAR)start,
(RU32)( end - start ),
&toSkip,
&longestLength,
&isUnicode ) &&
_MIN_SAMPLE_STR_LEN <= longestLength &&
_MAX_SAMPLE_STR_LEN >= longestLength )
{
if( rpal_bloom_addIfNew( stringsSeen,
start,
longestLength ) )
{
if( obsLib_addPattern( sample,
start,
longestLength,
sampleNumber ) )
{
sampleNumber++;
}
}
}
start += toSkip;
}
}
}
rFile_close( hFile );
}
rpal_memory_free( scratch );
}
rpal_bloom_destroy( stringsSeen );
}
}
return sample;
}
void test_multiPattern(void)
{
HObs hObs = NULL;
RU8 pattern1[] = { 0x01, 0x02, 0x03, 0x04 };
RU8 pattern2[] = { 0x01, 0x02, 0x03, 0x06 };
RU8 pattern3[] = { 0x01, 0x02, 0x06, 0x04 };
RU8 pattern4[] = { 0xEF, 0x02, 0x03, 0x04 };
RU8 buffer1[] = { 0x02, 0x04, 0xFF, 0xEF,
0x01, 0x02, 0x03, 0x04 };
RU8 buffer2[] = { 0x02, 0x04, 0xFF, 0xEF,
0x01, 0x02, 0x03, 0x04, 0xEE, 0x6F };
RU8 buffer3[] = { 0x02, 0x04, 0xFF, 0xEF,
0x01, 0x02, 0x01, 0x04, 0xEE, 0x6F };
RU8 buffer4[] = { 0x02, 0x04, 0xFF, 0xEF, 0x01,
0x02, 0x03, 0x04, 0xEE, 0x6F,
0x01, 0x02, 0x03, 0x04 };
RU8 buffer5[] = { 0x02, 0x04, 0xFF, 0xEF,
0x02, 0x03, 0x04, 0x04,
0xEE, 0x6F, 0x01, 0x02, 0x03, 0x04 };
RU8 buffer6[] = { 0x02, 0x04, 0xFF, 0xEF,
0x02, 0x03, 0x04, 0x04,
0xEE, 0x6F, 0x01, 0x02,
0x03, 0x04, 0x01, 0x02, 0x06, 0x04 };
RU32 context1 = 0;
RU32 context2 = 0;
RU32 context3 = 0;
RU32 context4 = 0;
PVOID hitCtx = NULL;
RU8* hitLoc = NULL;
hObs = obsLib_new( 0, 0 );
CU_ASSERT_TRUE_FATAL( rpal_memory_isValid( hObs ) );
CU_ASSERT_TRUE_FATAL( obsLib_addPattern( hObs, (RPU8)&pattern1, sizeof( pattern1 ), &context1 ) );
CU_ASSERT_TRUE_FATAL( obsLib_addPattern( hObs, (RPU8)&pattern2, sizeof( pattern2 ), &context2 ) );
CU_ASSERT_TRUE_FATAL( obsLib_addPattern( hObs, (RPU8)&pattern3, sizeof( pattern3 ), &context3 ) );
CU_ASSERT_TRUE_FATAL( obsLib_addPattern( hObs, (RPU8)&pattern4, sizeof( pattern4 ), &context4 ) );
// 1 pattern found end of buffer
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer1, sizeof( buffer1 ) ) );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context1 );
CU_ASSERT_EQUAL( hitLoc, buffer1 + sizeof( buffer1 ) - 4 );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
// 1 pattern found middle of buffer
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer2, sizeof( buffer2 ) ) );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context1 );
CU_ASSERT_EQUAL( hitLoc, buffer2 + sizeof( buffer2 ) - 6 );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
// 0 pattern found
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer3, sizeof( buffer3 ) ) );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
// 2 pattern found end and middle of buffer
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer4, sizeof( buffer4 ) ) );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context1 );
CU_ASSERT_EQUAL( hitLoc, buffer4 + sizeof( buffer4 ) - 10 );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context1 );
CU_ASSERT_EQUAL( hitLoc, buffer4 + sizeof( buffer4 ) - 4 );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
// Multi 1
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer5, sizeof( buffer5 ) ) );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context4 );
CU_ASSERT_EQUAL( hitLoc, buffer5 + 3 );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context1 );
CU_ASSERT_EQUAL( hitLoc, buffer5 + sizeof( buffer5 ) - 4 );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
// Multi 2
CU_ASSERT_TRUE_FATAL( obsLib_setTargetBuffer( hObs, buffer6, sizeof( buffer6 ) ) );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context4 );
CU_ASSERT_EQUAL( hitLoc, buffer6 + 3 );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context1 );
CU_ASSERT_EQUAL( hitLoc, buffer6 + sizeof( buffer6 ) - 8 );
CU_ASSERT_TRUE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
CU_ASSERT_EQUAL( hitCtx, &context3 );
CU_ASSERT_EQUAL( hitLoc, buffer6 + sizeof( buffer6 ) - 4 );
CU_ASSERT_FALSE( obsLib_nextHit( hObs, &hitCtx, &hitLoc ) );
obsLib_free( hObs );
}