int main(int argc, char * argv[]){
const char * evals_text = "evals.text";
pinyin_option_t options = USE_TONE;
FacadeChewingTable largetable;
MemoryChunk * chunk = new MemoryChunk;
chunk->load("pinyin_index.bin");
largetable.load(options, chunk, NULL);
FacadePhraseTable2 phrase_table;
chunk = new MemoryChunk;
chunk->load("phrase_index.bin");
phrase_table.load(chunk, NULL);
FacadePhraseIndex phrase_index;
if (!load_phrase_index(&phrase_index))
exit(ENOENT);
Bigram system_bigram;
system_bigram.attach("bigram.db", ATTACH_READONLY);
Bigram user_bigram;
user_bigram.attach(NULL, ATTACH_CREATE|ATTACH_READWRITE);
PinyinLookup2 pinyin_lookup(options, &largetable, &phrase_index,
&system_bigram, &user_bigram);
/* open evals.text. */
FILE * evals_file = fopen(evals_text, "r");
if ( NULL == evals_file ) {
fprintf(stderr, "Can't open file:%s\n", evals_text);
exit(ENOENT);
}
PhraseTokens phrase_tokens;
memset(phrase_tokens, 0, sizeof(PhraseTokens));
phrase_index.prepare_tokens(phrase_tokens);
/* Evaluates the correction rate of test text documents. */
size_t tested_count = 0; size_t passed_count = 0;
char* linebuf = NULL; size_t size = 0;
TokenVector tokens = g_array_new(FALSE, TRUE, sizeof(phrase_token_t));
phrase_token_t token = null_token;
while( getline(&linebuf, &size, evals_file) ) {
if ( feof(evals_file) )
break;
if ( '\n' == linebuf[strlen(linebuf)-1] )
linebuf[strlen(linebuf)-1] = '\0';
glong phrase_len = 0;
ucs4_t * phrase = g_utf8_to_ucs4(linebuf, -1, NULL, &phrase_len, NULL);
token = null_token;
if ( 0 != phrase_len ) {
int result = phrase_table.search(phrase_len, phrase, phrase_tokens);
int num = get_first_token(phrase_tokens, token);
if ( !(result & SEARCH_OK) )
token = null_token;
g_free(phrase);
phrase = NULL;
}
if ( null_token == token ) {
if ( tokens->len ) { /* one test. */
if ( do_one_test(&pinyin_lookup, &phrase_index, tokens) ) {
tested_count ++; passed_count ++;
} else {
tested_count ++;
}
g_array_set_size(tokens, 0);
}
} else {
g_array_append_val(tokens, token);
}
}
if ( tokens->len ) { /* one test. */
if ( do_one_test(&pinyin_lookup, &phrase_index, tokens) ) {
tested_count ++; passed_count ++;
} else {
tested_count ++;
}
}
parameter_t rate = passed_count / (parameter_t) tested_count;
printf("correction rate:%f\n", rate);
g_array_free(tokens, TRUE);
fclose(evals_file);
free(linebuf);
phrase_index.destroy_tokens(phrase_tokens);
return 0;
}
int main(int argc, char * argv[]){
int i = 1;
bool train_pi_gram = true;
const char * bigram_filename = "deleted_bigram.db";
setlocale(LC_ALL, "");
while ( i < argc ){
if ( strcmp("--help", argv[i]) == 0){
print_help();
exit(0);
} else if ( strcmp("--skip-pi-gram-training", argv[i]) == 0 ){
train_pi_gram = false;
} else if ( strcmp("--deleted-bigram-file", argv[i]) == 0){
if ( ++i >= argc ) {
print_help();
exit(EINVAL);
}
bigram_filename = argv[i];
} else {
print_help();
exit(EINVAL);
}
++i;
}
/* load phrase table. */
PhraseLargeTable2 phrase_table;
MemoryChunk * new_chunk = new MemoryChunk;
new_chunk->load("phrase_index.bin");
phrase_table.load(new_chunk);
FacadePhraseIndex phrase_index;
if (!load_phrase_index(&phrase_index))
exit(ENODATA);
Bigram bigram;
bigram.attach(bigram_filename, ATTACH_CREATE|ATTACH_READWRITE);
char* linebuf = NULL; size_t size = 0;
phrase_token_t last_token, cur_token = last_token = 0;
while( getline(&linebuf, &size, stdin) ){
if ( feof(stdin) )
break;
if ( '\n' == linebuf[strlen(linebuf) - 1] ) {
linebuf[strlen(linebuf) - 1] = '\0';
}
TAGLIB_PARSE_SEGMENTED_LINE(&phrase_index, token, linebuf);
last_token = cur_token;
cur_token = token;
/* skip null_token in second word. */
if ( null_token == cur_token )
continue;
/* skip pi-gram training. */
if ( null_token == last_token ){
if ( !train_pi_gram )
continue;
last_token = sentence_start;
}
/* train bi-gram */
SingleGram * single_gram = NULL;
bigram.load(last_token, single_gram);
if ( NULL == single_gram ){
single_gram = new SingleGram;
}
guint32 freq, total_freq;
//increase freq
if (single_gram->get_freq(cur_token, freq))
assert(single_gram->set_freq(cur_token, freq + 1));
else
assert(single_gram->insert_freq(cur_token, 1));
//increase total freq
single_gram->get_total_freq(total_freq);
single_gram->set_total_freq(total_freq + 1);
bigram.store(last_token, single_gram);
delete single_gram;
}
free(linebuf);
return 0;
}
int main(int argc, char * argv[]){
FILE * input = stdin;
FILE * output = stdout;
setlocale(LC_ALL, "");
GError * error = NULL;
GOptionContext * context;
context = g_option_context_new("- shortest path segment");
g_option_context_add_main_entries(context, entries, NULL);
if (!g_option_context_parse(context, &argc, &argv, &error)) {
g_print("option parsing failed:%s\n", error->message);
exit(EINVAL);
}
if (outputfile) {
output = fopen(outputfile, "w");
if (NULL == output) {
perror("open file failed");
exit(EINVAL);
}
}
if (argc > 2) {
fprintf(stderr, "too many arguments.\n");
exit(EINVAL);
}
if (2 == argc) {
input = fopen(argv[1], "r");
if (NULL == input) {
perror("open file failed");
exit(EINVAL);
}
}
SystemTableInfo2 system_table_info;
bool retval = system_table_info.load(SYSTEM_TABLE_INFO);
if (!retval) {
fprintf(stderr, "load table.conf failed.\n");
exit(ENOENT);
}
/* init phrase table */
FacadePhraseTable3 phrase_table;
phrase_table.load(SYSTEM_PHRASE_INDEX, NULL);
/* init phrase index */
FacadePhraseIndex phrase_index;
const pinyin_table_info_t * phrase_files =
system_table_info.get_default_tables();
if (!load_phrase_index(phrase_files, &phrase_index))
exit(ENOENT);
CONTEXT_STATE state, next_state;
GArray * current_ucs4 = g_array_new(TRUE, TRUE, sizeof(ucs4_t));
PhraseTokens tokens;
memset(tokens, 0, sizeof(PhraseTokens));
phrase_index.prepare_tokens(tokens);
char * linebuf = NULL; size_t size = 0; ssize_t read;
while( (read = getline(&linebuf, &size, input)) != -1 ){
if ( '\n' == linebuf[strlen(linebuf) - 1] ) {
linebuf[strlen(linebuf) - 1] = '\0';
}
/* check non-ucs4 characters. */
const glong num_of_chars = g_utf8_strlen(linebuf, -1);
glong len = 0;
ucs4_t * sentence = g_utf8_to_ucs4(linebuf, -1, NULL, &len, NULL);
if ( len != num_of_chars ) {
fprintf(stderr, "non-ucs4 characters encountered:%s.\n", linebuf);
fprintf(output, "%d \n", null_token);
continue;
}
/* only new-line persists. */
if ( 0 == num_of_chars ) {
fprintf(output, "%d \n", null_token);
continue;
}
state = CONTEXT_INIT;
int result = phrase_table.search( 1, sentence, tokens);
g_array_append_val( current_ucs4, sentence[0]);
if ( result & SEARCH_OK )
state = CONTEXT_SEGMENTABLE;
else
state = CONTEXT_UNKNOWN;
for ( int i = 1; i < num_of_chars; ++i) {
int result = phrase_table.search( 1, sentence + i, tokens);
if ( result & SEARCH_OK )
next_state = CONTEXT_SEGMENTABLE;
else
next_state = CONTEXT_UNKNOWN;
if ( state == next_state ){
g_array_append_val(current_ucs4, sentence[i]);
continue;
}
assert ( state != next_state );
if ( state == CONTEXT_SEGMENTABLE )
deal_with_segmentable(&phrase_table, &phrase_index,
current_ucs4, output);
if ( state == CONTEXT_UNKNOWN )
deal_with_unknown(current_ucs4, output);
/* save the current character */
g_array_set_size(current_ucs4, 0);
g_array_append_val(current_ucs4, sentence[i]);
state = next_state;
}
if ( current_ucs4->len ) {
/* this seems always true. */
if ( state == CONTEXT_SEGMENTABLE )
deal_with_segmentable(&phrase_table, &phrase_index,
current_ucs4, output);
if ( state == CONTEXT_UNKNOWN )
deal_with_unknown(current_ucs4, output);
g_array_set_size(current_ucs4, 0);
}
/* print extra enter */
if ( gen_extra_enter )
fprintf(output, "%d \n", null_token);
g_free(sentence);
}
phrase_index.destroy_tokens(tokens);
/* print enter at file tail */
fprintf(output, "%d \n", null_token);
g_array_free(current_ucs4, TRUE);
free(linebuf);
fclose(input);
fclose(output);
return 0;
}
int main(int argc, char * argv[]){
setlocale(LC_ALL, "");
SystemTableInfo system_table_info;
bool retval = system_table_info.load("../../data/table.conf");
if (!retval) {
fprintf(stderr, "load table.conf failed.\n");
exit(ENOENT);
}
/* init phrase table */
FacadePhraseTable2 phrase_table;
MemoryChunk * chunk = new MemoryChunk;
chunk->load("../../data/phrase_index.bin");
phrase_table.load(chunk, NULL);
const pinyin_table_info_t * phrase_files =
system_table_info.get_table_info();
/* init phrase index */
FacadePhraseIndex phrase_index;
if (!load_phrase_index(phrase_files, &phrase_index))
exit(ENOENT);
/* init bi-gram */
Bigram system_bigram;
system_bigram.attach("../../data/bigram.db", ATTACH_READONLY);
Bigram user_bigram;
gfloat lambda = system_table_info.get_lambda();
/* init phrase lookup */
PhraseLookup phrase_lookup(lambda,
&phrase_table, &phrase_index,
&system_bigram, &user_bigram);
/* try one sentence */
char * linebuf = NULL;
size_t size = 0;
ssize_t read;
while( (read = getline(&linebuf, &size, stdin)) != -1 ){
if ( '\n' == linebuf[strlen(linebuf) - 1] ) {
linebuf[strlen(linebuf) - 1] = '\0';
}
if ( strcmp ( linebuf, "quit" ) == 0)
break;
/* check non-ucs4 characters */
const glong num_of_chars = g_utf8_strlen(linebuf, -1);
glong len = 0;
ucs4_t * sentence = g_utf8_to_ucs4(linebuf, -1, NULL, &len, NULL);
if ( len != num_of_chars ) {
fprintf(stderr, "non-ucs4 characters are not accepted.\n");
g_free(sentence);
continue;
}
try_phrase_lookup(&phrase_lookup, sentence, len);
g_free(sentence);
}
free(linebuf);
return 0;
}
int main(int argc, char * argv[]){
int i = 1;
bool gen_extra_enter = false;
setlocale(LC_ALL, "");
/* deal with options */
while ( i < argc ){
if ( strcmp ("--help", argv[i]) == 0 ){
print_help();
exit(0);
} else if ( strcmp("--generate-extra-enter", argv[i]) == 0 ){
gen_extra_enter = true;
} else {
print_help();
exit(EINVAL);
}
++i;
}
/* init phrase table */
FacadePhraseTable2 phrase_table;
MemoryChunk * chunk = new MemoryChunk;
chunk->load("phrase_index.bin");
phrase_table.load(chunk, NULL);
/* init phrase index */
FacadePhraseIndex phrase_index;
if (!load_phrase_index(&phrase_index))
exit(ENOENT);
/* init bi-gram */
Bigram system_bigram;
system_bigram.attach("bigram.db", ATTACH_READONLY);
Bigram user_bigram;
/* init phrase lookup */
PhraseLookup phrase_lookup(&phrase_table, &phrase_index,
&system_bigram, &user_bigram);
CONTEXT_STATE state, next_state;
GArray * current_ucs4 = g_array_new(TRUE, TRUE, sizeof(ucs4_t));
PhraseTokens tokens;
memset(tokens, 0, sizeof(PhraseTokens));
phrase_index.prepare_tokens(tokens);
/* split the sentence */
char * linebuf = NULL; size_t size = 0; ssize_t read;
while( (read = getline(&linebuf, &size, stdin)) != -1 ){
if ( '\n' == linebuf[strlen(linebuf) - 1] ) {
linebuf[strlen(linebuf) - 1] = '\0';
}
/* check non-ucs4 characters */
const glong num_of_chars = g_utf8_strlen(linebuf, -1);
glong len = 0;
ucs4_t * sentence = g_utf8_to_ucs4(linebuf, -1, NULL, &len, NULL);
if ( len != num_of_chars ) {
fprintf(stderr, "non-ucs4 characters encountered:%s.\n", linebuf);
printf("\n");
continue;
}
/* only new-line persists. */
if ( 0 == num_of_chars ) {
printf("\n");
continue;
}
state = CONTEXT_INIT;
int result = phrase_table.search( 1, sentence, tokens);
g_array_append_val( current_ucs4, sentence[0]);
if ( result & SEARCH_OK )
state = CONTEXT_SEGMENTABLE;
else
state = CONTEXT_UNKNOWN;
for ( int i = 1; i < num_of_chars; ++i) {
int result = phrase_table.search( 1, sentence + i, tokens);
if ( result & SEARCH_OK )
next_state = CONTEXT_SEGMENTABLE;
else
next_state = CONTEXT_UNKNOWN;
if ( state == next_state ){
g_array_append_val(current_ucs4, sentence[i]);
continue;
}
assert ( state != next_state );
if ( state == CONTEXT_SEGMENTABLE )
deal_with_segmentable(&phrase_lookup, current_ucs4);
if ( state == CONTEXT_UNKNOWN )
deal_with_unknown(current_ucs4);
/* save the current character */
g_array_set_size(current_ucs4, 0);
g_array_append_val(current_ucs4, sentence[i]);
state = next_state;
}
if ( current_ucs4->len ) {
/* this seems always true. */
if ( state == CONTEXT_SEGMENTABLE )
deal_with_segmentable(&phrase_lookup, current_ucs4);
if ( state == CONTEXT_UNKNOWN )
deal_with_unknown(current_ucs4);
g_array_set_size(current_ucs4, 0);
}
/* print extra enter */
if ( gen_extra_enter )
printf("\n");
}
phrase_index.destroy_tokens(tokens);
/* print enter at file tail */
printf("\n");
g_array_free(current_ucs4, TRUE);
free(linebuf);
return 0;
}
int main( int argc, char * argv[]){
pinyin_option_t options =
USE_TONE | USE_RESPLIT_TABLE | PINYIN_CORRECT_ALL | PINYIN_AMB_ALL;
FacadeChewingTable largetable;
MemoryChunk * chunk = new MemoryChunk;
chunk->load("../../data/pinyin_index.bin");
largetable.load(options, chunk, NULL);
FacadePhraseIndex phrase_index;
if (!load_phrase_index(&phrase_index))
exit(ENOENT);
Bigram system_bigram;
system_bigram.attach("../../data/bigram.db", ATTACH_READONLY);
Bigram user_bigram;
user_bigram.attach(NULL, ATTACH_CREATE|ATTACH_READWRITE);
PinyinLookup2 pinyin_lookup(options, &largetable, &phrase_index,
&system_bigram, &user_bigram);
/* prepare the prefixes for get_best_match. */
TokenVector prefixes = g_array_new
(FALSE, FALSE, sizeof(phrase_token_t));
g_array_append_val(prefixes, sentence_start);
CandidateConstraints constraints = g_array_new
(TRUE, FALSE, sizeof(lookup_constraint_t));
MatchResults results = g_array_new(FALSE, FALSE, sizeof(phrase_token_t));
char* linebuf = NULL; size_t size = 0; ssize_t read;
while( (read = getline(&linebuf, &size, stdin)) != -1 ){
if ( '\n' == linebuf[strlen(linebuf) - 1] ) {
linebuf[strlen(linebuf) - 1] = '\0';
}
if ( strcmp ( linebuf, "quit" ) == 0)
break;
FullPinyinParser2 parser;
ChewingKeyVector keys = g_array_new(FALSE, FALSE, sizeof(ChewingKey));
ChewingKeyRestVector key_rests =
g_array_new(FALSE, FALSE, sizeof(ChewingKeyRest));
parser.parse(options, keys, key_rests, linebuf, strlen(linebuf));
if ( 0 == keys->len ) /* invalid pinyin */
continue;
/* initialize constraints. */
g_array_set_size(constraints, keys->len);
for ( size_t i = 0; i < constraints->len; ++i){
lookup_constraint_t * constraint = &g_array_index(constraints, lookup_constraint_t, i);
constraint->m_type = NO_CONSTRAINT;
}
guint32 start_time = record_time();
for ( size_t i = 0; i < bench_times; ++i)
pinyin_lookup.get_best_match(prefixes, keys, constraints, results);
print_time(start_time, bench_times);
for ( size_t i = 0; i < results->len; ++i){
phrase_token_t * token = &g_array_index(results, phrase_token_t, i);
if ( null_token == *token)
continue;
printf("pos:%ld,token:%d\t", i, *token);
}
printf("\n");
char * sentence = NULL;
pinyin_lookup.convert_to_utf8(results, sentence);
printf("%s\n", sentence);
g_array_free(keys, TRUE);
g_array_free(key_rests, TRUE);
g_free(sentence);
}
g_array_free(prefixes, TRUE);
g_array_free(constraints, TRUE);
g_array_free(results, TRUE);
free(linebuf);
return 0;
}
int main(int argc, char * argv[]){
setlocale(LC_ALL, "");
GError * error = NULL;
GOptionContext * context;
context = g_option_context_new("- generate deleted n-gram");
g_option_context_add_main_entries(context, entries, NULL);
if (!g_option_context_parse(context, &argc, &argv, &error)) {
g_print("option parsing failed:%s\n", error->message);
exit(EINVAL);
}
SystemTableInfo2 system_table_info;
bool retval = system_table_info.load(SYSTEM_TABLE_INFO);
if (!retval) {
fprintf(stderr, "load table.conf failed.\n");
exit(ENOENT);
}
FacadePhraseIndex phrase_index;
const pinyin_table_info_t * phrase_files =
system_table_info.get_default_tables();
if (!load_phrase_index(phrase_files, &phrase_index))
exit(ENODATA);
Bigram bigram;
bigram.attach(bigram_filename, ATTACH_CREATE|ATTACH_READWRITE);
char* linebuf = NULL; size_t size = 0;
phrase_token_t last_token, cur_token = last_token = 0;
while( getline(&linebuf, &size, stdin) ){
if ( feof(stdin) )
break;
if ( '\n' == linebuf[strlen(linebuf) - 1] ) {
linebuf[strlen(linebuf) - 1] = '\0';
}
TAGLIB_PARSE_SEGMENTED_LINE(&phrase_index, token, linebuf);
last_token = cur_token;
cur_token = token;
/* skip null_token in second word. */
if ( null_token == cur_token )
continue;
/* skip pi-gram training. */
if ( null_token == last_token ){
if ( !train_pi_gram )
continue;
last_token = sentence_start;
}
/* train bi-gram */
SingleGram * single_gram = NULL;
bigram.load(last_token, single_gram);
if ( NULL == single_gram ){
single_gram = new SingleGram;
}
guint32 freq, total_freq;
//increase freq
if (single_gram->get_freq(cur_token, freq))
assert(single_gram->set_freq(cur_token, freq + 1));
else
assert(single_gram->insert_freq(cur_token, 1));
//increase total freq
single_gram->get_total_freq(total_freq);
single_gram->set_total_freq(total_freq + 1);
bigram.store(last_token, single_gram);
delete single_gram;
}
free(linebuf);
return 0;
}
