//check the specified string is make up with numeric. FRISO_API int friso_numeric_string( friso_charset_t charset, char *buffer ) { if ( charset == FRISO_UTF8 ) return utf8_numeric_string( buffer ); else if ( charset == FRISO_GBK ) return gbk_numeric_string( buffer ); return 0; }
//get the next latin word from the current position. __STATIC_API__ friso_hits_t next_basic_latin( friso_t friso, friso_task_t task ) { char __convert = 0, t = 0; string_buffer_t sb, temp; lex_entry_t e = NULL; //full-half width and upper-lower case exchange. task->unicode = get_utf8_unicode( task->buffer ); ___LATAIN_FULL_UPPER_CHECK___ //creat a new string buffer and append the task->buffer insite. sb = new_string_buffer_with_string( task->buffer ); //segmentation. while ( ( task->bytes = read_next_word( task, &task->idx, task->buffer ) ) != 0 ) { task->unicode = get_utf8_unicode( task->buffer ); if ( utf8_whitespace( task->unicode ) ) break; if ( utf8_en_punctuation( task->unicode ) && ! utf8_keep_punctuation( task->buffer ) ) break; if ( ! ( utf8_halfwidth_letter_digit( task->unicode ) || utf8_fullwidth_letter_digit( task->unicode ) ) ) { task->idx -= task->bytes; t = 1; break; } //full-half width and upper-lower case convert ___LATAIN_FULL_UPPER_CHECK___ //append the word the buffer. string_buffer_append( sb, task->buffer ); } /*clear the useless english punctuation from the end of the buffer.*/ for ( ; sb->length > 0 && sb->buffer[ sb->length - 1 ] != '%' && is_en_punctuation( sb->buffer[ sb->length - 1 ] ); ) { sb->buffer[ --sb->length ] = '\0'; } /* * find the chinese or english mixed word. * or single chinese units.*/ if ( t == 1 ) { if ( utf8_cjk_string( task->unicode ) ) { //temp string buffer. temp = new_string_buffer_with_string( sb->buffer ); for ( t = 0; t < friso->mix_len && ( task->bytes = read_next_word( task , &task->idx, task->buffer ) ) != 0; t++ ) { task->unicode = get_utf8_unicode( task->buffer ); if ( ! utf8_cjk_string( task->unicode ) ) { task->idx -= task->bytes; break; } string_buffer_append( temp, task->buffer ); //check the mixed word dictionary. if ( friso_dic_match( friso->dic, __LEX_MIX_WORDS__, temp->buffer ) ) { __convert = 1; //get the lexicon entry from the dictionary. e = friso_dic_get( friso->dic, __LEX_MIX_WORDS__, temp->buffer ); } } //correct the segmentation offset. task->idx -= ( temp->length - ( e == NULL ? sb->length : e->length ) ); free_string_buffer( temp ); //no match for mix word, try to find a single chinese unit. if ( __convert == 0 ) { //check if it is string made up with numeric if ( utf8_numeric_string( sb->buffer ) && ( task->bytes = read_next_word( task, &task->idx, task->buffer ) ) != 0 ) { //check the single chinese units dictionary. if ( friso_dic_match( friso->dic, __LEX_CJK_UNITS__, task->buffer ) ) { string_buffer_append( sb, task->buffer ); } else { task->idx -= task->bytes; } } } //end convert condition } } if ( __convert == 1 ) { free_string_buffer( sb ); task->hits->word = e->word; task->hits->type = __FRISO_SYS_WORDS__; } else { /* * adjust the string buffer. * here we do not trim the buffer cause its allocations will be free * after the call of friso_next - sooner or later it will be released. * if your memory almost run out, you should call string_buffer_trim. * or we save the time to do the allocations and copy the buffer insite. */ //string_buffer_trim( sb ); task->hits->word = string_buffer_devote( sb ); task->hits->type = __FRISO_NEW_WORDS__; } return task->hits; }