int creat(Head *head)
{
int data;
Head p = NULL;
while(1)
{
printf("please input data:");
scanf("%d", &data);
if(data <= 0)
{
break;
}
//head_insert(head, data);
middle_insert(head, data);
//tail_insert(head, data, &p);
}
return 0;
}
static void recursive_insert(lm_trie_t *trie, ngram_raw_t **raw_ngrams, uint32 *counts, int order)
{
uint32 unigram_idx = 0;
uint32 *words;
float *probs;
const uint32 unigram_count = (uint32)counts[0];
priority_queue_t *ngrams = priority_queue_create(order, &ngram_ord_comparator);
ngram_raw_ord_t *ngram;
uint32 *raw_ngrams_ptr;
int i;
words = (uint32 *)ckd_calloc(order, sizeof(*words)); //for blanks catching
probs = (float *)ckd_calloc(order - 1, sizeof(*probs)); //for blanks prob generating
ngram = (ngram_raw_ord_t *)ckd_calloc(1, sizeof(*ngram));
ngram->order = 1;
ngram->instance.words = &unigram_idx;
priority_queue_add(ngrams, ngram);
raw_ngrams_ptr = (uint32 *)ckd_calloc(order - 1, sizeof(*raw_ngrams_ptr));
for (i = 2; i <= order; ++i) {
ngram_raw_ord_t *tmp_ngram = (ngram_raw_ord_t *)ckd_calloc(1, sizeof(*tmp_ngram));
tmp_ngram->order = i;
raw_ngrams_ptr[i-2] = 0;
tmp_ngram->instance = raw_ngrams[i - 2][raw_ngrams_ptr[i-2]];
priority_queue_add(ngrams, tmp_ngram);
}
for (;;) {
ngram_raw_ord_t *top = (ngram_raw_ord_t *)priority_queue_poll(ngrams);
if (top->order == 1) {
trie->unigrams[unigram_idx].next = unigram_next(trie, order);
words[0] = unigram_idx;
probs[0] = trie->unigrams[unigram_idx].prob;
if (++unigram_idx == unigram_count + 1) {
ckd_free(top);
break;
}
priority_queue_add(ngrams, top);
} else {
for (i = 0; i < top->order - 1; i++) {
if (words[i] != top->instance.words[i]) {
//need to insert dummy suffixes to make ngram of higher order reachable
int j;
assert(i > 0); //unigrams are not pruned without removing ngrams that contains them
for (j = i; j < top->order - 1; j++) {
middle_t *middle = &trie->middle_begin[j - 1];
bitarr_address_t address = middle_insert(middle, top->instance.words[j], j + 1, order);
//calculate prob for blank
float calc_prob = probs[j - 1] + trie->unigrams[top->instance.words[j]].bo;
probs[j] = calc_prob;
lm_trie_quant_mwrite(trie->quant, address, j - 1, calc_prob, 0.0f);
}
}
}
memcpy(words, top->instance.words, top->order * sizeof(*words));
if (top->order == order) {
float *weights = top->instance.weights;
bitarr_address_t address = longest_insert(trie->longest, top->instance.words[top->order - 1]);
lm_trie_quant_lwrite(trie->quant, address, weights[0]);
} else {
float *weights = top->instance.weights;
middle_t *middle = &trie->middle_begin[top->order - 2];
bitarr_address_t address = middle_insert(middle, top->instance.words[top->order - 1], top->order, order);
//write prob and backoff
probs[top->order - 1] = weights[0];
lm_trie_quant_mwrite(trie->quant, address, top->order - 2, weights[0], weights[1]);
}
raw_ngrams_ptr[top->order - 2]++;
if (raw_ngrams_ptr[top->order - 2] < counts[top->order - 1]) {
top->instance = raw_ngrams[top->order-2][raw_ngrams_ptr[top->order - 2]];
priority_queue_add(ngrams, top);
} else {
ckd_free(top);
}
}
}
assert(priority_queue_size(ngrams) == 0);
priority_queue_free(ngrams, NULL);
ckd_free(raw_ngrams_ptr);
ckd_free(words);
ckd_free(probs);
}
