static int
ngrams_raw_read_order(ngram_raw_t ** raw_ngrams, lineiter_t ** li,
                      hash_table_t * wid, logmath_t * lmath, uint32 count,
                      int order, int order_max)
{
    char expected_header[20];
    uint32 i;

    sprintf(expected_header, "\\%d-grams:", order);
    while (*li && strcmp((*li)->buf, expected_header) != 0) {
	*li = lineiter_next(*li);
    }
    
    if (*li == NULL) {
	E_ERROR("Failed to find '%s', language model file truncated\n", expected_header);
	return -1;
    }
    
    *raw_ngrams = (ngram_raw_t *) ckd_calloc(count, sizeof(ngram_raw_t));
    for (i = 0; i < count; i++) {
        if (read_ngram_instance(li, wid, lmath, order, order_max,
                            &((*raw_ngrams)[i])) < 0)
            break;
    }

    qsort(*raw_ngrams, count, sizeof(ngram_raw_t), &ngram_ord_comparator);
    return 0;
}
Exemple #2
0
static void
ngrams_raw_read_order(ngram_raw_t ** raw_ngrams, lineiter_t ** li,
                      hash_table_t * wid, logmath_t * lmath, uint32 count,
                      int order, int order_max)
{
    char expected_header[20];
    uint32 i;

    sprintf(expected_header, "\\%d-grams:", order);
    while (*li && strcmp((*li)->buf, expected_header) != 0) {
        *li = lineiter_next(*li);
    }

    *raw_ngrams = (ngram_raw_t *) ckd_calloc(count, sizeof(ngram_raw_t));
    for (i = 0; i < count; i++) {
        read_ngram_instance(li, wid, lmath, order, order_max,
                            &((*raw_ngrams)[i]));
    }

    //sort raw ngrams that was read
    ngram_comparator(NULL, &order);     //setting up order in comparator
    qsort(*raw_ngrams, count, sizeof(ngram_raw_t), &ngram_comparator);
}