QcacheEntry_c::QcacheEntry_c()
: m_iIndexId ( -1 )
, m_tmStarted ( sphMicroTimer() )
, m_iElapsedMsec ( 0 )
, m_Key ( 0 )
, m_iMruPrev ( -1 )
, m_iMruNext ( -1 )
, m_iTotalMatches ( 0 )
, m_uLastDocid ( 0 )
{
}
int main ()
{
// threads should be initialized before memory allocations
char cTopOfMainStack;
sphThreadInit();
MemorizeStack ( &cTopOfMainStack );
CSphString sError;
CSphDictSettings tDictSettings;
ISphTokenizer * pTok = sphCreateUTF8Tokenizer();
CSphDict * pDict = sphCreateDictionaryCRC ( tDictSettings, pTok, sError, "rt1" );
CSphSource * pSrc = SpawnSource ( "SELECT id, channel_id, UNIX_TIMESTAMP(published) published, title, UNCOMPRESS(content) content FROM posting WHERE id<=10000 AND id%2=0", pTok, pDict );
ISphTokenizer * pTok2 = sphCreateUTF8Tokenizer();
CSphDict * pDict2 = sphCreateDictionaryCRC ( tDictSettings, pTok, sError, "rt2" );
CSphSource * pSrc2 = SpawnSource ( "SELECT id, channel_id, UNIX_TIMESTAMP(published) published, title, UNCOMPRESS(content) content FROM posting WHERE id<=10000 AND id%2=1", pTok2, pDict2 );
CSphSchema tSrcSchema;
if ( !pSrc->UpdateSchema ( &tSrcSchema, sError ) )
sphDie ( "update-schema failed: %s", sError.cstr() );
CSphSchema tSchema; // source schema must be all dynamic attrs; but index ones must be static
tSchema.m_dFields = tSrcSchema.m_dFields;
for ( int i=0; i<tSrcSchema.GetAttrsCount(); i++ )
tSchema.AddAttr ( tSrcSchema.GetAttr(i), false );
CSphConfigSection tRTConfig;
sphRTInit();
sphRTConfigure ( tRTConfig, true );
SmallStringHash_T< CSphIndex * > dTemp;
sphReplayBinlog ( dTemp, 0 );
ISphRtIndex * pIndex = sphCreateIndexRT ( tSchema, "testrt", 32*1024*1024, "data/dump", false );
pIndex->SetTokenizer ( pTok ); // index will own this pair from now on
pIndex->SetDictionary ( pDict );
if ( !pIndex->Prealloc ( false, false, sError ) )
sphDie ( "prealloc failed: %s", pIndex->GetLastError().cstr() );
g_pIndex = pIndex;
// initial indexing
int64_t tmStart = sphMicroTimer();
SphThread_t t1, t2;
sphThreadCreate ( &t1, IndexingThread, pSrc );
sphThreadCreate ( &t2, IndexingThread, pSrc2 );
sphThreadJoin ( &t1 );
sphThreadJoin ( &t2 );
#if 0
// update
tParams.m_sQuery = "SELECT id, channel_id, UNIX_TIMESTAMP(published) published, title, UNCOMPRESS(content) content FROM rt2 WHERE id<=10000";
SetupIndexing ( pSrc, tParams );
DoIndexing ( pSrc, pIndex );
#endif
// search
DoSearch ( pIndex );
// shutdown index (should cause dump)
int64_t tmShutdown = sphMicroTimer();
#if SPH_ALLOCS_PROFILER
printf ( "pre-shutdown allocs=%d, bytes="INT64_FMT"\n", sphAllocsCount(), sphAllocBytes() );
#endif
SafeDelete ( pIndex );
#if SPH_ALLOCS_PROFILER
printf ( "post-shutdown allocs=%d, bytes="INT64_FMT"\n", sphAllocsCount(), sphAllocBytes() );
#endif
int64_t tmEnd = sphMicroTimer();
printf ( "shutdown done in %d.%03d sec\n", (int)((tmEnd-tmShutdown)/1000000), (int)(((tmEnd-tmShutdown)%1000000)/1000) );
printf ( "total with shutdown %d.%03d sec, %.2f MB/sec\n",
(int)((tmEnd-tmStart)/1000000), (int)(((tmEnd-tmStart)%1000000)/1000),
g_fTotalMB*1000000.0f/(tmEnd-tmStart) );
#if SPH_DEBUG_LEAKS || SPH_ALLOCS_PROFILER
sphAllocsStats();
#endif
#if USE_WINDOWS
PROCESS_MEMORY_COUNTERS pmc;
HANDLE hProcess = OpenProcess ( PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, GetCurrentProcessId() );
if ( hProcess && GetProcessMemoryInfo ( hProcess, &pmc, sizeof(pmc)) )
{
printf ( "--- peak-wss=%d, peak-pagefile=%d\n", (int)pmc.PeakWorkingSetSize, (int)pmc.PeakPagefileUsage );
}
#endif
SafeDelete ( pIndex );
sphRTDone ();
}
void DoIndexing ( CSphSource * pSrc, ISphRtIndex * pIndex )
{
CSphString sError;
CSphVector<DWORD> dMvas;
int64_t tmStart = sphMicroTimer ();
int64_t tmAvgCommit = 0;
int64_t tmMaxCommit = 0;
int iCommits = 0;
for ( ;; )
{
if ( !pSrc->IterateDocument ( sError ) )
sphDie ( "iterate-document failed: %s", sError.cstr() );
ISphHits * pHitsNext = pSrc->IterateHits ( sError );
if ( !sError.IsEmpty() )
sphDie ( "iterate-hits failed: %s", sError.cstr() );
if ( pSrc->m_tDocInfo.m_iDocID )
pIndex->AddDocument ( pHitsNext, pSrc->m_tDocInfo, NULL, dMvas, sError );
if ( ( pSrc->GetStats().m_iTotalDocuments % COMMIT_STEP )==0 || !pSrc->m_tDocInfo.m_iDocID )
{
int64_t tmCommit = sphMicroTimer();
pIndex->Commit ();
tmCommit = sphMicroTimer()-tmCommit;
iCommits++;
tmAvgCommit += tmCommit;
tmMaxCommit = Max ( tmMaxCommit, tmCommit );
if ( !pSrc->m_tDocInfo.m_iDocID )
{
tmAvgCommit /= iCommits;
break;
}
}
if (!( pSrc->GetStats().m_iTotalDocuments % 100 ))
printf ( "%d docs\r", (int)pSrc->GetStats().m_iTotalDocuments );
static bool bOnce = true;
if ( iCommits*COMMIT_STEP>=5000 && bOnce )
{
printf ( "\n" );
DoSearch ( pIndex );
bOnce = false;
}
}
pSrc->Disconnect();
int64_t tmEnd = sphMicroTimer ();
float fTotalMB = (float)pSrc->GetStats().m_iTotalBytes/1000000.0f;
printf ( "commit-step %d, %d docs, %d bytes, %d.%03d sec, %.2f MB/sec\n",
COMMIT_STEP,
(int)pSrc->GetStats().m_iTotalDocuments,
(int)pSrc->GetStats().m_iTotalBytes,
(int)((tmEnd-tmStart)/1000000), (int)(((tmEnd-tmStart)%1000000)/1000),
fTotalMB*1000000.0f/(tmEnd-tmStart) );
printf ( "commit-docs %d, avg %d.%03d msec, max %d.%03d msec\n", COMMIT_STEP,
(int)(tmAvgCommit/1000), (int)(tmAvgCommit%1000),
(int)(tmMaxCommit/1000), (int)(tmMaxCommit%1000) );
g_fTotalMB += fTotalMB;
}
void DoIndexing ( CSphSource_MySQL * pSrc, ISphRtIndex * pIndex )
{
CSphString sError, sWarning, sFilter;
CSphVector<DWORD> dMvas;
int64_t tmStart = sphMicroTimer ();
int64_t tmAvgCommit = 0;
int64_t tmMaxCommit = 0;
int iCommits = 0;
for ( ;; )
{
const char ** pFields = (const char **)pSrc->NextDocument ( sError );
if ( !pFields )
break;
if ( pSrc->m_tDocInfo.m_uDocID )
pIndex->AddDocument ( pIndex->CloneIndexingTokenizer(), g_iFieldsCount, pFields, pSrc->m_tDocInfo, false, sFilter, NULL, dMvas, sError, sWarning, NULL );
if ( ( pSrc->GetStats().m_iTotalDocuments % COMMIT_STEP )==0 || !pSrc->m_tDocInfo.m_uDocID )
{
int64_t tmCommit = sphMicroTimer();
pIndex->Commit ( NULL, NULL );
tmCommit = sphMicroTimer()-tmCommit;
iCommits++;
tmAvgCommit += tmCommit;
tmMaxCommit = Max ( tmMaxCommit, tmCommit );
if ( !pSrc->m_tDocInfo.m_uDocID )
{
tmAvgCommit /= iCommits;
break;
}
}
if (!( pSrc->GetStats().m_iTotalDocuments % 100 ))
printf ( "%d docs\r", (int)pSrc->GetStats().m_iTotalDocuments );
static bool bOnce = true;
if ( iCommits*COMMIT_STEP>=5000 && bOnce )
{
printf ( "\n" );
DoSearch ( pIndex );
bOnce = false;
}
}
pSrc->Disconnect();
int64_t tmEnd = sphMicroTimer ();
float fTotalMB = (float)pSrc->GetStats().m_iTotalBytes/1000000.0f;
printf ( "commit-step %d, %d docs, %d bytes, %d.%03d sec, %.2f MB/sec\n",
COMMIT_STEP,
(int)pSrc->GetStats().m_iTotalDocuments,
(int)pSrc->GetStats().m_iTotalBytes,
(int)((tmEnd-tmStart)/1000000), (int)(((tmEnd-tmStart)%1000000)/1000),
fTotalMB*1000000.0f/(tmEnd-tmStart) );
printf ( "commit-docs %d, avg %d.%03d msec, max %d.%03d msec\n", COMMIT_STEP,
(int)(tmAvgCommit/1000), (int)(tmAvgCommit%1000),
(int)(tmMaxCommit/1000), (int)(tmMaxCommit%1000) );
g_fTotalMB += fTotalMB;
}