/* Append the line to the transcript window. */
static wchar_t *write_to_ts_win(wchar_t *str, int max_col, int max_row, WINDOW *win, wchar_t *buffer, int *line, int *buffer_size)
{
int size = sizeof(wchar_t) * (wcslen(str)+(max_col*4));
wchar_t *text = (wchar_t*)malloc(size);
int i;
/* Copy over string. */
memset(text, '\0', size);
wcscpy(text, str);
/* Pad out the string so it takes up the whole line. */
while( wcslen(text) % max_col != 0 ) {
wcscat(text, L" ");
}
/* ---- */
for(i = 0; i < s_wcslen(text); i ++) {
wcscat(text, L" ");
}
/* Deal with buffer over run for the transcript window. */
while( (wcslen(buffer) + wcslen(text)) > *buffer_size ) {
int i = 0;
wchar_t *temp_buffer = (wchar_t*)malloc( sizeof(wchar_t) * (*buffer_size*2) );
/* Copy over old values. */
for(i = 0; i < wcslen(buffer); i++) {
temp_buffer[i] = buffer[i];
}
free(buffer); /* Free old buffer. */
*buffer_size *= 2;
buffer = temp_buffer;
}
/* Append text to transcript window. */
wcscat(buffer, text);
if(user_is_scrolling == 0) {
print_buffer_to_window(win, max_col*max_row, max_col, buffer, line);
} else {
window_page_up(win, line, max_col, buffer);
}
// log_writeln(text);
free(text);
return buffer;
}
/*
Function: iSrchFilterProcessTerm()
Purpose: This function processes the passed term, stripping the field
name (if present), setting the field ID bit map if needed
and stemming the term
Parameters: pssSrchSearch search structure
psiSrchIndex index structure
pvLngStemmer stemmer
pwcTerm term
pucFieldIDBitmap field ID bitmap (optional)
uiFieldIDBitmapLength field ID bitmap length (optional)
ppwcTerm return pointer for the processed term
pbFieldIDBitmapSet return flag set to true if the field ID bitmap was set
Globals: none
Returns: SRCH error code
*/
static int iSrchFilterProcessTerm
(
struct srchSearch *pssSrchSearch,
struct srchIndex *psiSrchIndex,
void *pvLngStemmer,
wchar_t *pwcTerm,
unsigned char *pucFieldIDBitmap,
unsigned int uiFieldIDBitmapLength,
wchar_t **ppwcTerm,
boolean *pbFieldIDBitmapSet
)
{
int iError = SRCH_NoError;
unsigned char pucFieldName[SRCH_INFO_SYMBOL_MAXIMUM_LENGTH + 1] = {'\0'};
unsigned int uiFieldID = 0;
unsigned int uiFieldOptions = SRCH_INFO_FIELD_OPTION_NONE;
boolean bFieldIDBitmapSet = false;
boolean bContainsUpperCase = false;
wchar_t *pwcPtr = NULL;
ASSERT(pssSrchSearch != NULL);
ASSERT(psiSrchIndex != NULL);
ASSERT(pvLngStemmer != NULL);
ASSERT(bUtlStringsIsStringNULL(pwcTerm) == false);
ASSERT(((pucFieldIDBitmap == NULL) && (uiFieldIDBitmapLength <= 0)) || ((pucFieldIDBitmap != NULL) && (uiFieldIDBitmapLength > 0)));
ASSERT(ppwcTerm != NULL);
ASSERT(pbFieldIDBitmapSet != NULL);
/* Set the field option defaults */
iSrchInfoGetFieldOptionDefaults(psiSrchIndex, &uiFieldOptions);
/* Extract field name if present */
if ( (bUtlStringsIsWideStringUrl(pwcTerm) == false) && ((pwcPtr = s_wcspbrk(pwcTerm, SRCH_FILTER_FIELD_NAME_SEPARATORS)) != NULL) ) {
wchar_t *pwcFieldNamePtr = NULL;
/* Set the field name */
pwcFieldNamePtr = pwcTerm;
/* Set the term */
pwcTerm = pwcPtr + 1;
/* Null terminate the field name, separating the term from the field name */
*pwcPtr = L'\0';
/* Convert the field name from wide characters to utf-8 */
if ( (iError = iLngConvertWideStringToUtf8_s(pwcFieldNamePtr, 0, pucFieldName, SRCH_INFO_SYMBOL_MAXIMUM_LENGTH + 1)) != LNG_NoError ) {
iUtlLogError(UTL_LOG_CONTEXT, "Failed to convert a field name from wide characters to utf-8, lng error: %d.", iError);
return (SRCH_FilterCharacterSetConvertionFailed);
}
/* Look up the field ID for this field */
if ( (iError = iSrchInfoGetFieldID(psiSrchIndex, pucFieldName, &uiFieldID)) == SRCH_NoError ) {
/* Get the field options, reset them to the default if we cant get them */
if ( (iError = iSrchInfoGetFieldInfo(psiSrchIndex, uiFieldID, NULL, 0, NULL, 0, NULL, &uiFieldOptions)) != SRCH_NoError ) {
/* Set the field option defaults */
iSrchInfoGetFieldOptionDefaults(psiSrchIndex, &uiFieldOptions);
}
/* Set the field ID in the bitmap */
if ( pucFieldIDBitmap != NULL ) {
/* Set the field ID in the bitmap - field ID 0 is not a field */
UTL_BITMAP_SET_BIT_IN_POINTER(pucFieldIDBitmap, uiFieldID - 1);
/* Field ID bit map is set */
bFieldIDBitmapSet = true;
}
}
}
/* Check if there is an unfielded search option if the field ID bit map was not set, and if the field name is not the wildcard field name */
if ( (bFieldIDBitmapSet == false) && ((pucFieldName == NULL) || (s_strcmp(pucFieldName, SRCH_PARSER_WILDCARD_FIELD_NAME_STRING) != 0)) ) {
unsigned char pucUnfieldedSearchFieldNames[SPI_FIELD_NAME_MAXIMUM_LENGTH + 1] = {"\0"};
/* Get the unfielded search field names and, if they exist, use them for this search */
if ( iSrchInfoGetUnfieldedSearchFieldNames(psiSrchIndex, pucUnfieldedSearchFieldNames, SPI_FIELD_NAME_MAXIMUM_LENGTH + 1) == SRCH_NoError ) {
unsigned char *pucUnfieldedSearchFieldNamePtr = NULL;
unsigned char *pucUnfieldedSearchFieldNamesStrtokPtr = NULL;
/* Loop parsing the unfielded search field names */
for ( pucUnfieldedSearchFieldNamePtr = (unsigned char *)s_strtok_r(pucUnfieldedSearchFieldNames, SRCH_INFO_UNFIELDED_SEARCH_FIELD_NAMES_SEPARATORS, (char **)&pucUnfieldedSearchFieldNamesStrtokPtr);
pucUnfieldedSearchFieldNamePtr != NULL;
pucUnfieldedSearchFieldNamePtr = (unsigned char *)s_strtok_r(NULL, SRCH_INFO_UNFIELDED_SEARCH_FIELD_NAMES_SEPARATORS, (char **)&pucUnfieldedSearchFieldNamesStrtokPtr) ) {
/* Set the field ID in the bitmap */
if ( iSrchInfoGetFieldID (psiSrchIndex, pucUnfieldedSearchFieldNamePtr, &uiFieldID) == SRCH_NoError ) {
ASSERT(uiFieldID <= uiFieldIDBitmapLength);
/* Set the field ID in the bitmap, we know there are fields other than field ID 0
** since there are unfielded search field names - field ID 0 is not a field
*/
UTL_BITMAP_SET_BIT_IN_POINTER(pucFieldIDBitmap, uiFieldID - 1);
/* Field ID bit map is set */
bFieldIDBitmapSet = true;
}
}
}
}
/* Set the mixed case flag, which really means 'does this term contain upper case, or non-alphanumerics' */
bContainsUpperCase = bLngCaseDoesWideStringContainUpperCase(pwcTerm);
/* Stem the term if we are stemming on this field and if the term does not end in a wildcard */
if ( (bSrchInfoFieldOptionStemming(uiFieldOptions) == true) && (bContainsUpperCase == false) && (s_wcschr(SRCH_PARSER_WILDCARDS_WSTRING , pwcTerm[s_wcslen(pwcTerm) - 1]) == NULL) ) {
if ( (iError = iLngStemmerStemTerm(pvLngStemmer, pwcTerm, 0)) != LNG_NoError ) {
iUtlLogError(UTL_LOG_CONTEXT, "Failed to stem a term, lng error: %d.", iError);
return (SRCH_FeedbackStemmingFailed);
}
}
/* Set the return pointers */
*ppwcTerm = pwcTerm;
*pbFieldIDBitmapSet = bFieldIDBitmapSet;
return (SRCH_NoError);
}
/*
Function: iPrsParseTextFile()
Purpose: Adds term to the index for a given file.
Parameters: pppPrsParser structure containing the various parser options
we use to parse the data
ppfPrsFormat file profile pointing to the various file options
we use to parse the data
pucFilePath file path to be parsed
pfInputFile data stream to be parsed
pfIndexerFile structured index stream output file
pfDataLength return pointer for the data length parsed
Globals: none
Returns: 0 on success, -1 on error
*/
int iPrsParseTextFile
(
struct prsParser *pppPrsParser,
struct prsFormat *ppfPrsFormat,
unsigned char *pucFilePath,
FILE *pfInputFile,
FILE *pfIndexerFile,
float *pfDataLength
)
{
int iError = UTL_NoError;
int iStatus = 0;
FILE *pfLocalFile = NULL;
unsigned char *pucLine = NULL;
unsigned char *pucNormalizedLine = NULL;
wchar_t *pwcLine = NULL;
unsigned int uiLineLength = 0;
unsigned int ulLineCapacity = 0;
unsigned int uiNormalizedLineCapacity = 0;
wchar_t *pwcParseLinePtr = NULL;
off_t zCurrentOffset = 0;
off_t zStartOffset = 0;
off_t zEndOffset = 0;
unsigned int uiTermPosition = 0;
unsigned char pucTrueFilePath[UTL_FILE_PATH_MAX + 1] = {'\0'};
unsigned int uiLanguageID = LNG_LANGUAGE_ANY_ID;
unsigned int uiFieldID = 0;
boolean bIndexLine = false;
boolean bParseLine = false;
boolean bTermPositions = false;
boolean bIndexing = false;
boolean bSetStartCharacter = true;
boolean bFinishDocument = false;
float fLineLength = 0;
float fDataLength = 0;
/* Check the parameters */
if ( pppPrsParser == NULL ) {
iUtlLogError(UTL_LOG_CONTEXT, "Null 'pppPrsParser' parameter passed to 'iPrsParseTextFile'.");
return (-1);
}
if ( ppfPrsFormat == NULL ) {
iUtlLogError(UTL_LOG_CONTEXT, "Null 'ppfPrsFormat' parameter passed to 'iPrsParseTextFile'.");
return (-1);
}
if ( pfIndexerFile == NULL ) {
iUtlLogError(UTL_LOG_CONTEXT, "Null 'pfIndexerFile' parameter passed to 'iPrsParseTextFile'.");
return (-1);
}
if ( (bUtlStringsIsStringNULL(pucFilePath) == true) && (pfInputFile == NULL) ) {
iUtlLogError(UTL_LOG_CONTEXT, "Null or empty 'pucFilePath' &'pfInputFile' parameters passed to 'iPrsParseTextFile'.");
return (-1);
}
if ( (bUtlStringsIsStringNULL(pucFilePath) == false) && (pfInputFile != NULL) ) {
iUtlLogError(UTL_LOG_CONTEXT, "Invalid 'pucFilePath' &'pfInputFile' parameters passed to 'iPrsParseTextFile'.");
return (-1);
}
if ( pfDataLength == NULL ) {
iUtlLogError(UTL_LOG_CONTEXT, "Null 'pfDataLength' parameter passed to 'iPrsParseTextFile'.");
return (-1);
}
/* Preset the data length return pointer */
*pfDataLength = 0;
/* Open the file path if it was provided */
if ( bUtlStringsIsStringNULL(pucFilePath) == false ) {
/* Get the real file path of this file */
if ( (iError = iUtlFileGetTruePath(pucFilePath, pucTrueFilePath, UTL_FILE_PATH_MAX + 1)) != UTL_NoError ) {
iUtlLogError(UTL_LOG_CONTEXT, "Failed to get the true path of the file path: '%s', utl error: %d.", pucFilePath, iError);
iStatus = -1;
goto bailFromlPrsParseTextFile;
}
/* Open the file to index */
if ( (pfLocalFile = s_fopen(pucTrueFilePath, "r")) == NULL ) {
/* Send a message
** M message
*/
if ( fprintf(pfIndexerFile, "M File: '%s' could not be opened.\n", pucTrueFilePath) < 0 ) {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to send a message line to the indexer");
}
iStatus = -1;
goto bailFromlPrsParseTextFile;
}
/* Send a message to the indexer
** M (Message to be displayed by the indexer)
*/
if ( pppPrsParser->bSuppressMessages == false ) {
if ( fprintf(pfIndexerFile, "M Parsing file: '%s'.\n", pucTrueFilePath) < 0 ) {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to send a message line to the indexer");
}
}
}
/* Otherwise read from the input file */
else if ( pfInputFile != NULL ) {
pfLocalFile = pfInputFile;
}
/* Otherwise croak */
else {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to find data either from a file path or an input file stream");
}
/* Initialize for the first document */
zCurrentOffset = 0;
zStartOffset = 0;
zEndOffset = 0;
uiTermPosition = 1;
bIndexing = false;
bSetStartCharacter = true;
bFinishDocument = false;
/* Loop around reading every line in the file */
while ( true ) {
boolean bEoF = false;
unsigned char *pucLineEndPtr = NULL;
/* Reset the line to an empty string, note that pucLine is dynamically
** allocated so we make sure that it is allocated before doing so
*/
if ( pucLine != NULL ) {
pucLine[0] = '\0';
}
/* Reset the line length */
uiLineLength = 0;
/* This ugly little loop reads stuff from the file we are indexing making sure that
** it reads an entire actual line (ie not limited by the fact that fgets() can only
** read fixed length buffers) and erases any end of line characters
*/
do {
/* Allocate/reallocate lines if needed */
if ( (pucLine == NULL) || (ulLineCapacity == 0) || ((uiLineLength + BUFSIZ) > ulLineCapacity) ) {
unsigned char *pucLinePtr = NULL;
/* Extend the line */
if ( (pucLinePtr = (unsigned char *)s_realloc(pucLine, (size_t)(sizeof(unsigned char) * (ulLineCapacity + BUFSIZ)))) == NULL ) {
iStatus = -1;
goto bailFromlPrsParseTextFile;
}
/* Hand over the pointer */
pucLine = pucLinePtr;
/* Clear the new allocation */
s_memset(pucLine + ulLineCapacity, 0, BUFSIZ);
/* Set the line capacity */
ulLineCapacity += BUFSIZ;
/* Extend the normalized line if normalization is supported */
if ( pppPrsParser->pvLngUnicodeNormalizer != NULL ) {
unsigned char *pucNormalizedLinePtr = NULL;
/* Set the normalized line capacity */
uiNormalizedLineCapacity = ulLineCapacity * LNG_UNICODE_NORMALIZATION_EXPANSION_MULTIPLIER_MAX;
/* Extend the normalized line */
if ( (pucNormalizedLinePtr = (unsigned char *)s_realloc(pucNormalizedLine, (size_t)(sizeof(unsigned char) * uiNormalizedLineCapacity))) == NULL ) {
iStatus = -1;
goto bailFromlPrsParseTextFile;
}
/* Hand over the pointer */
pucNormalizedLine = pucNormalizedLinePtr;
}
/* Extend the wide character line */
{
wchar_t *pwcLinePtr = NULL;
/* Extend the wide character line */
if ( (pwcLinePtr = (wchar_t *)s_realloc(pwcLine, (size_t)(sizeof(wchar_t) * ulLineCapacity))) == NULL ) {
iStatus = -1;
goto bailFromlPrsParseTextFile;
}
/* Hand over the pointer */
pwcLine = pwcLinePtr;
}
}
/* Read the next line chunk, appending to the current line and setting the end of file flag */
bEoF = (s_fgets(pucLine + uiLineLength, ulLineCapacity - uiLineLength, pfLocalFile) == NULL) ? true : false;
/* Get the new line length here for optimization */
uiLineLength = s_strlen(pucLine);
/* Erase the trailing new line - be platform sensitive, handle PC first, then Unix and Mac */
if ( (uiLineLength >= 2) && (pucLine[uiLineLength - 2] == '\r') ) {
pucLineEndPtr = pucLine + (uiLineLength - 2);
uiLineLength -= 2;
}
else if ( (uiLineLength >= 1) && ((pucLine[uiLineLength - 1] == '\n') || (pucLine[uiLineLength - 1] == '\r')) ) {
pucLineEndPtr = pucLine + (uiLineLength - 1);
uiLineLength -= 1;
}
else if ( (uiLineLength >= 1) && (bEoF == true) ) {
pucLineEndPtr = pucLine + uiLineLength;
}
else {
pucLineEndPtr = NULL;
}
if ( pucLineEndPtr != NULL ) {
*pucLineEndPtr = '\0';
}
} while ( (pucLineEndPtr == NULL) && (bEoF == false) );
/* Increment the line length */
fLineLength++;
/* Increment the data length */
fDataLength += uiLineLength;
/* Set the current character */
zCurrentOffset += uiLineLength;
/* Check to see if we have reached the end of the file, if we have we finish the document and bail */
if ( (bEoF == true) && (pucLineEndPtr == NULL) ) {
/* Finish the document if we are indexing */
if ( bIndexing == true ) {
/* Finish the document */
if ( iPrsFinishDocument(pppPrsParser, ppfPrsFormat, pfIndexerFile, pucTrueFilePath, zStartOffset, zCurrentOffset - 1, uiTermPosition - 1) != 0 ) {
iStatus = -1;
goto bailFromlPrsParseTextFile;
}
}
break;
}
/* Reset the wide character line */
if ( pwcLine != NULL ) {
pwcLine[0] = L'\0';
}
/* Set the parse line pointer */
pwcParseLinePtr = pwcLine;
/* Convert the line from its native character set to wide characters */
if ( bUtlStringsIsStringNULL(pucLine) == false ) {
/* Set the line pointer */
unsigned char *pucLinePtr = pucLine;
/* Use a separate variable for the line length */
unsigned int uiLineAllocatedByteLength = ulLineCapacity * sizeof(wchar_t);
/* Clean the unicode if requested */
if ( pppPrsParser->bCleanUnicode == true ) {
/* Clean the unicode */
if ( (iError = iLngUnicodeCleanUtf8String(pucLinePtr, PRS_UNICODE_REPLACEMENT_CHARACTER)) != LNG_NoError ) {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to clean the string, lng error: %d, string: '%s'", iError, pucLinePtr);
}
}
/* Normalize the line if the unicode normalizer is enabled */
if ( pppPrsParser->pvLngUnicodeNormalizer != NULL ) {
/* Normalize the line */
if ( (iError = iLngUnicodeNormalizeString(pppPrsParser->pvLngUnicodeNormalizer, pucLinePtr, uiLineLength, &pucNormalizedLine, &uiNormalizedLineCapacity)) != LNG_NoError ) {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to normalize the string, lng error: %d, string: '%s'", iError, pucLinePtr);
}
/* Set the line pointer */
pucLinePtr = pucNormalizedLine;
uiLineLength = s_strlen(pucLinePtr);
}
/* Convert the line from utf-8 to wide character, note the conversion error handling */
if ( (iError = iLngConverterConvertString(pppPrsParser->pvLngConverter, ((pppPrsParser->bSkipUnicodeErrors == true) ? LNG_CONVERTER_SKIP_ON_ERROR : LNG_CONVERTER_RETURN_ON_ERROR),
pucLinePtr, uiLineLength, (unsigned char **)&pwcLine, &uiLineAllocatedByteLength)) != LNG_NoError ) {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to convert the string from utf-8 set to wide characters, lng error: %d, string: '%s'", iError, pucLinePtr);
}
/* Set the parse line pointer */
pwcParseLinePtr = pwcLine;
}
/* Check for a document start if a start function was supplied */
if ( (ppfPrsFormat->bPrsDocumentStartFunction != NULL) && (ppfPrsFormat->bPrsDocumentStartFunction(pwcParseLinePtr) == true) ) {
/* We need to finish the current document if we are currently indexing */
if ( bIndexing == true ) {
/* Set the end character if it has not been set, this occurs if we have
** both a start function and an end function, and we detect a new document
** start without having first detected a document end
*/
if ( zEndOffset <= zStartOffset ) {
zEndOffset = (zCurrentOffset - uiLineLength) - 1;
}
/* Finish the document, we do this here because we dont want to process the current line, it belongs to the new document */
if ( iPrsFinishDocument(pppPrsParser, ppfPrsFormat, pfIndexerFile, pucTrueFilePath, zStartOffset, zEndOffset, uiTermPosition - 1) != 0 ) {
iStatus = -1;
goto bailFromlPrsParseTextFile;
}
/* Check to see if we have reached the document count limit and bail if we have */
if ( (pppPrsParser->uiDocumentCountMax > 0) && (pppPrsParser->uiDocumentCount >= pppPrsParser->uiDocumentCountMax) ) {
break;
}
/* Initialize for the next document */
uiTermPosition = 1;
}
else {
/* Turn on indexing */
bIndexing = true;
}
/* Set the start character */
zStartOffset = zCurrentOffset - uiLineLength;
bSetStartCharacter = false;
}
/* We are always indexing if there is no start function, because if we
** reach the end of a document, we are automatically indexing the next one
*/
else if ( ppfPrsFormat->bPrsDocumentStartFunction == NULL ) {
/* Make sure indexing is turned on */
bIndexing = true;
/* Set the start character if needed */
if ( bSetStartCharacter == true ) {
zStartOffset = zCurrentOffset - uiLineLength;
bSetStartCharacter = false;
}
}
/* Check for a document end if an end function was supplied */
if ( (ppfPrsFormat->bPrsDocumentEndFunction != NULL) && (ppfPrsFormat->bPrsDocumentEndFunction(pwcParseLinePtr) == true) && (fLineLength > 1) ) {
/* Finish the document if we run into a document end */
if ( bIndexing == true ) {
/* Request that we set a new start character, finish the document, and turn off indexing */
bSetStartCharacter = true;
bFinishDocument = true;
bIndexing = false;
/* Set the end character */
zEndOffset = zCurrentOffset - 1;
}
}
/* No end function */
else if ( ppfPrsFormat->bPrsDocumentEndFunction == NULL ) {
/* Set the end character */
zEndOffset = zCurrentOffset - 1;
}
/* Just loop to read the next line if we are not indexing and we dont need to finish a document */
if ( (bIndexing == false) && (bFinishDocument == false) ) {
continue;
}
/* Clear the parameters */
uiLanguageID = LNG_LANGUAGE_ANY_ID;
uiFieldID = 0;
bIndexLine = true;
bParseLine = true;
bTermPositions = true;
/* Process the document line */
if ( ppfPrsFormat->vPrsDocumentLineFunction != NULL ) {
ppfPrsFormat->vPrsDocumentLineFunction(pwcParseLinePtr, &uiLanguageID, &uiFieldID, &bIndexLine, &bParseLine, &bTermPositions);
}
/* Index the contents of the line */
if ( bIndexLine == true ) {
/* Parse the line */
if ( bParseLine == true ) {
/* Parse the text line */
if ( iPrsParseTextLine(pppPrsParser, ppfPrsFormat, pfIndexerFile, pwcParseLinePtr, pwcParseLinePtr + s_wcslen(pwcParseLinePtr),
uiLanguageID, uiFieldID, (bTermPositions == true) ? &uiTermPosition : NULL) != 0 ) {
if ( fprintf(pfIndexerFile, "M Failed to add the document terms.\n") < 0 ) {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to send a message line to the indexer");
}
}
}
/* Get a term from the line and add */
else {
wchar_t *pwcLinePtr = NULL;
/* Find the start of the term */
for ( pwcLinePtr = pwcParseLinePtr; *pwcLinePtr != L'\0'; pwcLinePtr++ ) {
if ( iswalnum(*pwcLinePtr) != 0 ) {
break;
}
}
/* Is the term long enough to add? */
if ( s_wcslen(pwcLinePtr) >= pppPrsParser->uiTermLengthMinimum ) {
/* Send the language line if the language is known */
if ( uiLanguageID != LNG_LANGUAGE_ANY_ID ) {
unsigned char pucLanguageCode[LNG_LANGUAGE_CODE_LENGTH + 1] = {'\0'};
if ( iLngGetLanguageCodeFromID(uiLanguageID, pucLanguageCode, LNG_LANGUAGE_CODE_LENGTH + 1) == LNG_NoError ) {
/* Send the language
** L language{A}
*/
if ( fprintf(pfIndexerFile, "L %s\n", pucLanguageCode) < 0 ) {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to send a language line to the indexer");
}
}
}
/* Send the term to the indexer */
if ( iPrsSendTermToIndexer(pppPrsParser, ppfPrsFormat, pfIndexerFile, pwcLinePtr, pwcLinePtr + s_wcslen(pwcLinePtr),
uiFieldID, (bTermPositions == true) ? uiTermPosition : 0) != 0 ) {
if ( fprintf(pfIndexerFile, "M Failed to add a document term.\n") < 0 ) {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to send a message line to the indexer");
}
}
/* Increment the term position */
if ( bTermPositions == true ) {
uiTermPosition++;
}
}
}
}
/* Finish the document if we were requested to do so */
if ( bFinishDocument == true ) {
/* Finish the current document */
if ( iPrsFinishDocument(pppPrsParser, ppfPrsFormat, pfIndexerFile, pucTrueFilePath, zStartOffset, zEndOffset, uiTermPosition - 1) != 0 ) {
if ( fprintf(pfIndexerFile, "M Failed to finish adding a document.\n") < 0 ) {
iUtlLogPanic(UTL_LOG_CONTEXT, "Failed to send a message line to the indexer");
}
}
/* Check to see if we have reached the document count limit */
if ( (pppPrsParser->uiDocumentCountMax > 0) && (pppPrsParser->uiDocumentCount >= pppPrsParser->uiDocumentCountMax) ) {
break;
}
/* Initialize for the next document */
uiTermPosition = 1;
/* Turn off the flag requesting that we finish the document */
bFinishDocument = false;
}
} /* while (true) */
/* Bail label */
bailFromlPrsParseTextFile:
/* Close the file descriptor if we opened it ourselves */
if ( bUtlStringsIsStringNULL(pucFilePath) == false ) {
s_fclose(pfLocalFile);
}
/* Free the line pointers */
s_free(pucLine);
s_free(pucNormalizedLine);
s_free(pwcLine);
/* Increment the data length return pointer */
*pfDataLength += fDataLength;
return (iStatus);
}
