void ApplyMorphology ( CSphIndex * pIndex )
{
CSphVector<BYTE> dInBuffer, dOutBuffer;
const int READ_BUFFER_SIZE = 1024;
dInBuffer.Reserve ( READ_BUFFER_SIZE );
char sBuffer[READ_BUFFER_SIZE];
while ( !feof(stdin) )
{
int iLen = fread ( sBuffer, 1, sizeof(sBuffer), stdin );
if ( !iLen )
break;
int iPos = dInBuffer.GetLength();
dInBuffer.Resize ( iPos+iLen );
memcpy ( &dInBuffer[iPos], sBuffer, iLen );
}
dInBuffer.Add(0);
dOutBuffer.Reserve ( dInBuffer.GetLength() );
CSphScopedPtr<ISphTokenizer> pTokenizer ( pIndex->GetTokenizer()->Clone ( SPH_CLONE_INDEX ) );
CSphDict * pDict = pIndex->GetDictionary();
BYTE * sBufferToDump = &dInBuffer[0];
if ( pTokenizer.Ptr() )
{
pTokenizer->SetBuffer ( &dInBuffer[0], dInBuffer.GetLength() );
while ( BYTE * sToken = pTokenizer->GetToken() )
{
if ( pDict )
pDict->ApplyStemmers ( sToken );
int iPos = dOutBuffer.GetLength();
int iLen = strlen ( (char *)sToken );
sToken[iLen] = ' ';
dOutBuffer.Resize ( iPos+iLen+1 );
memcpy ( &dOutBuffer[iPos], sToken, iLen+1 );
}
if ( dOutBuffer.GetLength() )
dOutBuffer[dOutBuffer.GetLength()-1] = 0;
else
dOutBuffer.Add(0);
sBufferToDump = &dOutBuffer[0];
}
fprintf ( stdout, "dumping stemmed results...\n%s\n", sBufferToDump );
}
