static int kor2chn_translator_analyze(index_word_extractor_t *extractor, index_word_t *index_word, int max)
{
kor2chn_translator_handle_t *handle=NULL;
token_t *token_array = NULL;
int32_t index_word_idx = 0;
int32_t num_of_tokens = 0;
int i=0;
if (extractor->id != 2) return MINUS_DECLINE;
handle = extractor->handle;
if (handle->done) return 0;
memset(&(index_word[0]), 0x00, sizeof(index_word_t));
token_array = handle->token_array;
handle->token_array = sb_realloc(token_array, max * sizeof(token_t));
token_array = handle->token_array;
num_of_tokens = sb_run_get_tokens(handle->tokenizer, token_array, max);
for(i=0; i < num_of_tokens && token_array[i].type != TOKEN_END_OF_DOCUMENT ; i++) {
switch(token_array[i].type)
{
case TOKEN_CHINESE:
/* TODO should be replaced by common/hanja.c:sb_hanja2hangul() */
CDconvChn2Kor(token_array[i].string , token_array[i].string);
// falls through
case TOKEN_ALPHABET:
strntoupper(token_array[i].string, MAX_WORD_LEN);
// falls through
case TOKEN_NUMBER:
case TOKEN_JAPANESE:
case TOKEN_KOREAN:
append_index_word( &(index_word[index_word_idx]),
&(token_array[i]), 1);
if (index_word[index_word_idx].len >= MAX_WORD_LEN-1)
goto FINISH;
break;
} // switch
} // loop
FINISH:
index_word_idx++;
handle->done = 1;
return index_word_idx;
}
static void near addripfile(char *szFilename)
{
if (ripfilebuf || (ripfilebuf=sb_new(STRBUFSZ))!=NULL)
{
if (!ripfilesent(szFilename))
{
if (sb_alloc(ripfilebuf,szFilename)==NULL) /* Out of space */
{
strbuf *newsb=sb_realloc(ripfilebuf,STRBUFSZ,0);
if (newsb==NULL)
sb_alloc(ripfilebuf,szFilename);
}
}
}
}