static bool
remove_low_priority_tokens(varray *tokens)
{
int i, j, to_remove[100], total_to_remove = 0;
varray *item;
vtoken *t;
for (i = 0; i < varray_length (tokens); i++)
{
item = varray_get (tokens, i);
if (varray_length (item) == 1)
continue;
for (j = 0; j < varray_length (item); j++)
{
t = varray_get (item, j);
if (t->priority <= VARNAM_TOKEN_PRIORITY_LOW) {
to_remove[total_to_remove++] = j;
}
}
for (j = 0; j < total_to_remove; j++)
{
varray_remove_at (item, to_remove[j] - j);
}
total_to_remove = 0;
if (get_total_possible_patterns (tokens) <= MAXIMUM_PATTERNS_TO_LEARN) {
return true;
}
}
return false;
}
static varray*
get_next_array(varray *tokens)
{
varray *item, *candidate = NULL;
int i;
bool same_priority = true;
vtoken *last, *lowest = NULL;
for (i = 0; i < varray_length (tokens); i++)
{
item = varray_get (tokens, i);
if (varray_length (item) == 1)
continue;
last = varray_get (item, varray_length (item) - 1);
if (lowest == NULL || last->priority < lowest->priority) {
lowest = last;
candidate = item;
}
if (last->priority != lowest->priority) {
same_priority = false;
}
}
if (same_priority)
{
candidate = get_largest_array (tokens);
if (varray_length (candidate) == 1)
return NULL;
}
return candidate;
}
static void AddEdit(MatchContext *ctx,int op,int off,int len)
{
DiffEdit *e;
if(len==0 || ctx->DiffEditArray==NULL)
{
return;
}
/* Add an edit to the SES(or
* coalesce if the op is the same)
*/
e=(DiffEdit *)varray_get(ctx->DiffEditArray,ctx->DiffEditArrayIndex);
if(e->op==op)
{
e->len += len;
}else if(e)
{
if(e->op)
{
ctx->DiffEditArrayIndex++;
e=(DiffEdit *)varray_get(ctx->DiffEditArray,ctx->DiffEditArrayIndex);
}
if(e)
{
e->op=op;
e->off=off;
e->len=len;
}
}
}
static void
apply_acceptance_condition(varray *tokens)
{
int i, j, to_remove[100], total_to_remove = 0, state, empty_arrays[100], empty_arrays_index = 0;
varray *item;
vtoken *t;
for (i = 0; i < varray_length (tokens); i++)
{
if (i == 0)
state = VARNAM_TOKEN_ACCEPT_IF_STARTS_WITH;
else if ((i + 1) == varray_length (tokens))
state = VARNAM_TOKEN_ACCEPT_IF_ENDS_WITH;
else
state = VARNAM_TOKEN_ACCEPT_IF_IN_BETWEEN;
item = varray_get (tokens, i);
for (j = 0; j < varray_length (item); j++)
{
t = varray_get (item, j);
switch (t->type)
{
case VARNAM_TOKEN_VIRAMA:
case VARNAM_TOKEN_VISARGA:
case VARNAM_TOKEN_ANUSVARA:
case VARNAM_TOKEN_NON_JOINER:
case VARNAM_TOKEN_NUMBER:
to_remove[total_to_remove++] = j;
break;
default:
if (varray_length (item) != 1 && t->accept_condition != VARNAM_TOKEN_ACCEPT_ALL && t->accept_condition != state) {
to_remove[total_to_remove++] = j;
}
break;
}
}
for (j = 0; j < total_to_remove; j++)
{
/* to_remove[j] - j is required to calculate the new index as deleting each item changes index */
varray_remove_at (item, to_remove[j] - j);
}
if (varray_length (item) == 0) {
/* This happens when all the items in this list is VIRAMA, VISARGA etc */
empty_arrays[empty_arrays_index++] = i;
}
total_to_remove = 0;
}
for (i = 0; i < empty_arrays_index; i++)
{
varray_remove_at (tokens, empty_arrays[i] - i);
}
}
static bool
can_learn_from_tokens (varnam *handle, varray *tokens, const char *word)
{
bool all_vowels = true, unknown_tokens = false;
int i, j, repeating_tokens = 0, last_token_id = 0;
vtoken *t, *unknown_token;
varray *array;
if (varray_length (tokens) < 2) {
set_last_error (handle, "Nothing to learn from '%s'", word);
return false;
}
for (i = 0; i < varray_length (tokens); i++)
{
array = varray_get (tokens, i);
for (j = 0; j < varray_length (array); j++)
{
t = varray_get (array, j);
if (t->type != VARNAM_TOKEN_VOWEL) all_vowels = false;
if (t->type == VARNAM_TOKEN_OTHER) {
unknown_tokens = true;
unknown_token = t;
goto done;
}
if (last_token_id == t->id) {
++repeating_tokens;
}
else {
repeating_tokens = 0;
last_token_id = t->id;
}
}
}
done:
if (all_vowels) {
set_last_error (handle, "Word contains only vowels. Nothing to learn from '%s'", word);
return false;
}
else if (unknown_tokens) {
set_last_error (handle, "Can't process '%s'. One or more characters in '%s' are not known", unknown_token->pattern, word);
return false;
}
else if (repeating_tokens >= 3) {
set_last_error (handle, "'%s' looks incorrect. Not learning anything", word);
return false;
}
return true;
}
struct eval *
eval_new(symlook_fn symlook, void *context)
{
struct eval *eval;
struct tok *em;
if ((eval = malloc(sizeof *eval)) == NULL) {
PMNO(errno);
return NULL;
}
memset(eval, 0, sizeof *eval);
if ((eval->toks = varray_new(sizeof(struct tok), NULL)) == NULL ||
(eval->opstk = stack_new(4096, NULL)) == NULL ||
(eval->stk = stack_new(4096, NULL)) == NULL ||
(em = varray_get(eval->toks, eval->toki++)) == NULL) {
AMSG("");
eval_del(eval);
return NULL;
}
eval->context = context;
eval->symlook = symlook;
em->type = TOK_TYPE_EMPTY;
stack_push(eval->opstk, em);
return eval;
}
void*
varray_get_last_item(varray *array)
{
assert (array);
return varray_get (array, array->index);
}
static int
next_tok(struct eval *eval, const tchar *src, const tchar *slim, struct tok **tok)
{
tchar buf[255];
int ret, i;
*tok = varray_get(eval->toks, eval->toki++);
if ((ret = next(src, slim, buf, buf + 255)) > 0) {
for (i = TOK_TYPE_BITOR; i < TOK_TYPE_VALUE; i++) {
if (*buf == ops[i]) {
break;
}
}
(*tok)->type = i;
if (i == TOK_TYPE_VALUE) {
if (istdigit(*buf)) {
(*tok)->val = tcstoul((const char *)buf, NULL, 0);
} else {
if (eval->symlook(buf, &(*tok)->val, eval->context) == -1) {
T1PMNF(errno = ENOENT, buf);
return -1;
}
}
}
} else {
(*tok)->type = TOK_TYPE_EMPTY;
}
return ret;
}
void model_draw(struct Model *model, float *mMat, float *vMat, float *pMat)
{
glEnable(GL_CULL_FACE);
glEnable(GL_DEPTH_TEST);
mat4 t_matrix;
mat4_identity(t_matrix);
mat4_mult(mMat, t_matrix, t_matrix);
mat4_mult(vMat, t_matrix, t_matrix);
mat4_mult(pMat, t_matrix, t_matrix);
//mat4_transpose(t_matrix);
int i;
for(i = 0; i < varray_length(&model->features); i++)
{
static int FALSE = 0;
const ModelFeature *feature = varray_get(&model->features, i);
glbProgramTexture(glbdrawmodel, GLB_FRAGMENT_SHADER, 0, feature->color);
glbProgramUniformMatrix(glbdrawmodel, GLB_VERTEX_SHADER, 0,
sizeof(int), true, &FALSE);
glbProgramUniformMatrix(glbdrawmodel, GLB_VERTEX_SHADER, 1,
sizeof(float[16]), true, t_matrix);
glbProgramDrawIndexed(glbdrawmodel, feature->mesh->vbuffer, feature->mesh->ibuffer);
}
}
static int _v(MatchContext *ctx,int k,int r)
{
int j;
j=k <= 0 ? -k * 4+r : k * 4 +(r - 2);
return *((int *)varray_get(ctx->buf,j));
}
void _varray_dump(varray_t* v)
{
int i;
for (i = 0; i < varray_length(v); i++)
{
printf("'%s', ", (char*)varray_get(v, i));
}
printf("\n");
}
static void _setv(MatchContext *ctx,int k,int r,int val)
{
int j;
int *i;
/* Pack -N to N into 0 to N * 2
*/
j=k <= 0 ? -k * 4+r : k * 4 +(r - 2);
i=(int *)varray_get(ctx->buf,j);
*i=val;
}
int
AdtInit(int verbose, struct cfg *cfg, char *args[])
{
char buf[0xFFFF];
struct allocator *suba;
struct stack s;
struct linkedlist l;
struct varray va;
if ((suba = suba_init(buf, 0xFFFF, 1, 0)) == NULL ||
stack_init(&s, 0, suba) == -1 ||
linkedlist_init(&l, 0, suba) == -1 ||
varray_init(&va, sizeof(int), suba) == -1) {
AMSG("");
return -1;
}
linkedlist_add(&l, "two");
stack_push(&s, "two");
linkedlist_add(&l, "three");
stack_push(&s, "one");
varray_get(&va, 444);
stack_push(&s, "three");
varray_get(&va, 4);
varray_get(&va, 44);
linkedlist_add(&l, "one");
if (varray_deinit(&va) != 0 ||
linkedlist_deinit(&l, NULL, NULL) != 0 ||
stack_deinit(&s, NULL, NULL) != 0) {
AMSG("");
return -1;
}
tcase_printf(verbose, "done ");
cfg = NULL;
args[0] = NULL;
return 0;
}
bool
varray_exists (varray *array, void *item, bool (*equals)(void *left, void *right))
{
int i;
for (i = 0; i < varray_length (array); i++)
{
if (equals(varray_get (array, i), item))
return true;
}
return false;
}
END_TEST
START_TEST(test_sort)
{
srand(12345);
int i, j;
uint64_t t1, t2, tt;
tt = 0;
int num_repeats = 10;
int max_size = 20000;
for(i = 0; i < num_repeats; i++)
{
varray_t* v = varray_new();
int size = _random64() % max_size;
for(j = 0; j < size; j++)
{
varray_add(v, _new_int64(_random64()));
}
t1 = gettimeusec();
varray_sort(v, _cmp_int64);
t2 = gettimeusec();
tt += t2 - t1;
for(j = 0; j < varray_length(v) - 1; j++)
{
fail_unless(_cmp_int64( varray_get(v, j), varray_get(v, j+1) ) <= 0, "array not sorted correctly");
}
for(j = varray_length(v) - 1; j >= 0; j--)
{
free( varray_get(v, j) );
varray_remove(v, j);
}
varray_free(v);
}
if (verbose) { printf("sort loop: %f /sec (%d repeats, %d array size)\n", (1000000*(double)num_repeats)/tt, num_repeats, max_size/2); }
}
void
varray_copy(varray *source, varray *destination)
{
int i;
void *item;
if (source == NULL) return;
if (destination == NULL) return;
for (i = 0; i < varray_length (source); i++) {
item = varray_get (source, i);
varray_push (destination, item);
}
}
static int
get_total_possible_patterns(varray *tokens)
{
int total = 1, i = 0;
varray *item;
for (i = 0; i < varray_length (tokens); i++)
{
item = varray_get (tokens, i);
total *= varray_length (item);
}
return total;
}
END_TEST
START_TEST (indic_digit_rendering)
{
int rc;
vword* word;
varray *words;
rc = varnam_transliterate (varnam_instance, "01", &words);
assert_success (rc);
ck_assert_int_eq (varray_length (words), 1);
word = varray_get (words, 0);
ck_assert_str_eq (word->text, "01");
rc = varnam_config (varnam_instance, VARNAM_CONFIG_USE_INDIC_DIGITS, 1);
assert_success (rc);
rc = varnam_transliterate (varnam_instance, "01", &words);
assert_success (rc);
ck_assert_int_eq (varray_length (words), 1);
word = varray_get (words, 0);
ck_assert_str_eq (word->text, "०१");
}
// expects a NULL-terminated string array
int _varray_cmp_to_string_array(varray_t* v, const char** strs)
{
int i = 0;
int result = 0;
while (strs[i] != NULL)
{
char* e = (char*)varray_get(v, i);
if (e == NULL) { result = -2; continue; }
if (result == 0) { result = strcmp(e, strs[i]); }
i++;
}
if (varray_length(v) != i) { result = -3; }
return result;
}
END_TEST
START_TEST (dependent_vowel_rendering)
{
int rc;
vword* word;
varray *words;
rc = varnam_transliterate(varnam_instance, "aaa", &words);
assert_success (rc);
ck_assert_int_eq (varray_length(words), 1);
word = varray_get(words, 0);
ck_assert_str_eq (word->text, "aa-value1a-value2");
}
END_TEST
START_TEST (cancellation_character_should_force_independent_vowel_form)
{
int rc;
vword* word;
varray *words;
rc = varnam_transliterate(varnam_instance, "aa_a", &words);
assert_success (rc);
ck_assert_int_eq (varray_length(words), 1);
word = varray_get(words, 0);
ck_assert_str_eq (word->text, "aa-value1a-value1");
}
static varray*
get_largest_array(varray *tokens)
{
int i;
varray *item = NULL, *largest = NULL;
for (i = 0; i < varray_length (tokens); i++)
{
item = varray_get (tokens, i);
if (largest == NULL || varray_length (item) > varray_length (largest)) {
largest = item;
}
}
return largest;
}
static
void _vroute_srvc_space_inspect(struct vroute_srvc_space* space, vroute_srvc_space_inspect_t cb, void* cookie, vnonce* nonce, uint32_t insp_id)
{
struct varray* services = NULL;
int i = 0;
int j = 0;
vassert(space);
vassert(insp_id);
for (i = 0; i < NBUCKETS; i++) {
services = &space->bucket[i].srvcs;
for (j = 0; j < varray_size(services); j++) {
cb((struct vservice*)varray_get(services, j), cookie, nonce, insp_id);
}
}
return ;
}
int GetStringSimilarity(const char *a,const char *b)
{
int n,m,d;
int sn,i;
struct varray ses;
varray_init(&ses,sizeof(DiffEdit),NULL);
n=strlen(a);
m=strlen(b);
if ((d=DiffArray(
a,0,n,
b,0,m,
NULL,0,&ses,&sn,NULL)) == -1)
{
return 0;
}
int match_len=0;
int unmatch_len=0;
for (i=0; i < sn; i++) {
DiffEdit *e=(DiffEdit *)varray_get(&ses,i);
switch (e->op)
{
case DIFF_MATCH:
//fwrite(a + e->off,1,e->len,stdout);
match_len+=e->len*2;
break;
case DIFF_INSERT:
unmatch_len+=e->len;
break;
case DIFF_DELETE:
unmatch_len+=e->len;
break;
}
}
if(match_len+unmatch_len==0)
{
return 100;
}
return (match_len*100)/(match_len+unmatch_len);
}
void
ensure_word_list_contains(varray *words, const char *word)
{
int i = 0, found = 0;
vword *w;
strbuf *error;
for (i = 0; i < varray_length (words); i++) {
w = varray_get (words, i);
if (strcmp (w->text, word) == 0) {
found = 1;
break;
}
}
if (!found) {
error = strbuf_init (50);
strbuf_addf (error, "Expected word list to contain '%s'", word);
ck_abort_msg (strbuf_to_s (error));
}
}
END_TEST
START_TEST (confidence_should_get_updated_for_existing_words)
{
int rc;
varray* words;
vword* word;
const char *word_to_learn = "കഖ";
rc = varnam_learn (varnam_instance, word_to_learn);
assert_success (rc);
rc = varnam_learn (varnam_instance, word_to_learn);
assert_success (rc);
rc = varnam_transliterate (varnam_instance, "kagha", &words);
assert_success (rc);
ck_assert_int_eq (varray_length (words), 2);
word = varray_get (words, 0);
ck_assert_int_eq (word->confidence, 2);
}
void
varray_free(varray *array, void (*destructor)(void*))
{
int i;
void *item;
if (array == NULL)
return;
if (destructor != NULL)
{
for(i = 0; i < varray_length(array); i++)
{
item = varray_get (array, i);
if (item != NULL) destructor(item);
}
}
if (array->memory != NULL)
free(array->memory);
free(array);
}
/*
* the routine to dump all service nodes in service routing table
*
* @space:
*/
static
void _vroute_srvc_space_dump(struct vroute_srvc_space* space, vdump_t dcb)
{
struct varray* services = NULL;
int titled = 0;
int i = 0;
int j = 0;
vassert(space);
for (i = 0; i < NBUCKETS; i++) {
services = &space->bucket[i].srvcs;
for (j = 0; j < varray_size(services); j++) {
if (!titled) {
dcb("-> list of services in service routing space:");
titled = 1;
}
vservice_dump((struct vservice*)varray_get(services, j), dcb);
}
}
return;
}
/*
* the routine to pack a @ping query and send it.
* @route:
* @conn:
*/
static
int _vroute_dht_ping(struct vroute* route, vnodeConn* conn)
{
struct vroute_recr_space* recr_space = &route->recr_space;
void* buf = NULL;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
retS((!(route->props & PROP_PING))); //ping disabled.
buf = vdht_buf_alloc();
retE((!buf));
vtoken_make(&token);
ret = route->enc_ops->ping(&token, &route->myid, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
route->ops->inspect(route, &token, VROUTE_INSP_SND_PING);
// make a record according to the query, which will be used to
// check invality of response msg.
recr_space->ops->make(recr_space, &token);
vlogD("send @ping");
return 0;
}
/*
* the routine to pack and send a response to @ping query back to source node
* where the ping query was from.
* @route:
* @conn:
* @token:
* @info :
*/
static
int _vroute_dht_ping_rsp(struct vroute* route, vnodeConn* conn, vtoken* token, vnodeInfo* nodei)
{
void* buf = NULL;
int ret = 0;
vassert(route);
vassert(conn);
vassert(token);
vassert(nodei);
retS((!(route->props & PROP_PING_R)));
buf = vdht_buf_alloc();
retE((!buf));
ret = route->enc_ops->ping_rsp(token, &route->myid, nodei, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
vlogD("send @ping_rsp");
return 0;
}
/*
* the routine to pack and send @find_node query to specific node to get infos
* of a given node.
* @route:
* @conn:
* @target
*/
static
int _vroute_dht_find_node(struct vroute* route, vnodeConn* conn, vnodeId* targetId)
{
struct vroute_recr_space* recr_space = &route->recr_space;
void* buf = NULL;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(targetId);
retS((!(route->props & PROP_FIND_NODE)));
buf = vdht_buf_alloc();
retE((!buf));
vtoken_make(&token);
ret = route->enc_ops->find_node(&token, &route->myid, targetId, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
route->ops->inspect(route, &token, VROUTE_INSP_SND_FIND_NODE);
recr_space->ops->make(recr_space, &token);
vlogD("send @find_node");
return 0;
}
/*
* the routine to pack and send a response to @find_node query.
* @route:
* @conn:
* @token:
* @info:
*/
static
int _vroute_dht_find_node_rsp(struct vroute* route, vnodeConn* conn, vtoken* token, vnodeInfo* nodei)
{
void* buf = NULL;
int ret = 0;
vassert(route);
vassert(conn);
vassert(token);
vassert(nodei);
retS((!(route->props & PROP_FIND_NODE_R)));
buf = vdht_buf_alloc();
retE((!buf));
ret = route->enc_ops->find_node_rsp(token, &route->myid, nodei, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
vlogD("send @find_node_rsp");
return 0;
}
/*
* the routine to pack and send a @find_closest_nodes query( closest to @targetId).
* @route:
* @conn:
* @targetId:
*/
static
int _vroute_dht_find_closest_nodes(struct vroute* route, vnodeConn* conn, vnodeId* targetId)
{
struct vroute_recr_space* recr_space = &route->recr_space;
void* buf = NULL;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(targetId);
retS((!(route->props & PROP_FIND_CLOSEST_NODES)));
buf = vdht_buf_alloc();
retE((!buf));
vtoken_make(&token);
ret = route->enc_ops->find_closest_nodes(&token, &route->myid, targetId, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
route->ops->inspect(route, &token, VROUTE_INSP_SND_FIND_CLOSEST_NODES);
recr_space->ops->make(recr_space, &token);
vlogD("send @find_closest_nodes");
return 0;
}
/*
* the routine to pack and send back a response to @find_closest_nodes query.
* @route:
* @conn:
* @token:
* @closest: array of closest nodes (closest to given id);
*/
static
int _vroute_dht_find_closest_nodes_rsp(struct vroute* route, vnodeConn* conn, vtoken* token, struct varray* closest)
{
void* buf = NULL;
int ret = 0;
vassert(route);
vassert(conn);
vassert(token);
vassert(closest);
retS((!(route->props & PROP_FIND_CLOSEST_NODES_R)));
buf = vdht_buf_alloc();
retE((!buf));
ret = route->enc_ops->find_closest_nodes_rsp(token, &route->myid, closest, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
vlogD("send @find_closest_nodes_rsp");
return 0;
}
/*
* the routine to pack and send a @reflex_addr query.
* @route:
* @conn:
*/
static
int _vroute_dht_reflex(struct vroute* route, vnodeConn* conn)
{
struct vroute_recr_space* recr_space = &route->recr_space;
void* buf = NULL;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
buf = vdht_buf_alloc();
retE((!buf));
vtoken_make(&token);
ret = route->enc_ops->reflex(&token, &route->myid, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
route->ops->inspect(route, &token, VROUTE_INSP_SND_REFLEX);
recr_space->ops->make(recr_space, &token);
vlogD("send @reflex");
return 0;
}
/*
* the routine to pack and send back a response to @reflex_addr query.
* @route:
* @conn:
* @token:
* @reflexive_addr:
*/
static
int _vroute_dht_reflex_rsp(struct vroute* route, vnodeConn* conn, vtoken* token, struct sockaddr_in* reflexive_addr)
{
void* buf = NULL;
int ret = 0;
vassert(route);
vassert(conn);
vassert(token);
vassert(reflexive_addr);
buf = vdht_buf_alloc();
retE((!buf));
ret = route->enc_ops->reflex_rsp(token, &route->myid, reflexive_addr, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
vlogD("send @reflex_rsp");
return 0;
}
/*
* the routine to pack and send a @probe_connectivity query
* @route:
* @conn:
* @targetId:
*/
static
int _vroute_dht_probe(struct vroute* route, vnodeConn* conn, vnodeId* targetId)
{
struct vroute_recr_space* recr_space = &route->recr_space;
void* buf = NULL;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(targetId);
buf = vdht_buf_alloc();
retE((!buf));
vtoken_make(&token);
ret = route->enc_ops->probe(&token, &route->myid, targetId, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
route->ops->inspect(route, &token, VROUTE_INSP_SND_PROBE);
recr_space->ops->make(recr_space, &token);
vlogD("send @probe");
return 0;
}
/*
* the routine to pack and send back a response to @probe_connectivity query.
* @route:
* @conn:
* @token:
*/
static
int _vroute_dht_probe_rsp(struct vroute* route, vnodeConn* conn, vtoken* token)
{
void* buf = NULL;
int ret = 0;
vassert(route);
vassert(conn);
vassert(token);
buf = vdht_buf_alloc();
retE((!buf));
ret = route->enc_ops->probe_rsp(token, &route->myid, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
vlogD("send @probe_rsp");
return 0;
}
/*
* the routine to pack and send @post_service indication.
* @route:
* @conn:
* @srvc:
*/
static
int _vroute_dht_post_service(struct vroute* route, vnodeConn* conn, vsrvcInfo* srvci)
{
void* buf = NULL;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(srvci);
retS((!(route->props & PROP_POST_SERVICE)));
buf = vdht_buf_alloc();
retE((!buf));
vtoken_make(&token);
ret = route->enc_ops->post_service(&token, &route->myid, srvci, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
vlogD("send @post_service");
return 0;
}
/*
* the routine to pack and send a @find_service query.
* @route:
* @conn:
* @srvcId:
*/
static
int _vroute_dht_find_service(struct vroute* route, vnodeConn* conn, vsrvcHash* hash)
{
struct vroute_recr_space* recr_space = &route->recr_space;
void* buf = NULL;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(hash);
retS((!(route->props & PROP_FIND_SERVICE)));
buf = vdht_buf_alloc();
retE((!buf));
vtoken_make(&token);
ret = route->enc_ops->find_service(&token, &route->myid, hash, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
route->ops->inspect(route, &token, VROUTE_INSP_SND_FIND_SERVICE);
recr_space->ops->make(recr_space, &token);
vlogD("send @find_service");
return 0;
}
/*
* the routine to pack and send a response to @find_service query.
* @route:
* @conn:
* @token:
* @srvcs:
*/
static
int _vroute_dht_find_service_rsp(struct vroute* route, vnodeConn* conn, vtoken* token, vsrvcInfo* srvc)
{
void* buf = NULL;
int ret = 0;
vassert(route);
vassert(conn);
vassert(token);
vassert(srvc);
retS((!(route->props & PROP_FIND_SERVICE_R)));
buf = vdht_buf_alloc();
retE((!buf));
ret = route->enc_ops->find_service_rsp(token, &route->myid, srvc, buf, vdht_buf_len());
ret1E((ret < 0), vdht_buf_free(buf));
{
struct vmsg_usr msg = {
.addr = to_vsockaddr_from_sin(&conn->remote),
.spec = to_vsockaddr_from_sin(&conn->local),
.msgId = VMSG_DHT,
.data = buf,
.len = ret
};
ret = route->msger->ops->push(route->msger, &msg);
ret1E((ret < 0), vdht_buf_free(buf));
}
vlogD("send @find_service_rsp");
return 0;
}
static
struct vroute_dht_ops route_dht_ops = {
.ping = _vroute_dht_ping,
.ping_rsp = _vroute_dht_ping_rsp,
.find_node = _vroute_dht_find_node,
.find_node_rsp = _vroute_dht_find_node_rsp,
.find_closest_nodes = _vroute_dht_find_closest_nodes,
.find_closest_nodes_rsp = _vroute_dht_find_closest_nodes_rsp,
.reflex = _vroute_dht_reflex,
.reflex_rsp = _vroute_dht_reflex_rsp,
.probe = _vroute_dht_probe,
.probe_rsp = _vroute_dht_probe_rsp,
.post_service = _vroute_dht_post_service,
.find_service = _vroute_dht_find_service,
.find_service_rsp = _vroute_dht_find_service_rsp
};
static
void _aux_vnodeInfo_free(void* item, void* cookie)
{
vnodeInfo_relax* nodei = (vnodeInfo_relax*)item;
vassert(nodei);
vnodeInfo_relax_free(nodei);
return ;
}
/* the routine to call when receiving a @ping query.
*
* @route:
* @conn:
* @ctxt: dht decoder context.
*/
static
int _vroute_cb_ping(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_node_space* node_space = &route->node_space;
vnodeInfo_relax nodei_relax;
vnodeInfo* nodei = (vnodeInfo*)&nodei_relax;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
ret =route->dec_ops->ping(ctxt, &token, &fromId);
retE((ret < 0));
vnodeInfo_relax_init(&nodei_relax, &fromId, vnodeVer_unknown(), 0);
vnodeInfo_add_addr(&nodei, &conn->remote);
ret = node_space->ops->add_node(node_space, nodei, 1);
retE((ret < 0));
route->ops->inspect(route, &token, VROUTE_INSP_RCV_PING);
ret = route->dht_ops->ping_rsp(route, conn, &token, (vnodeInfo*)&route->node->nodei);
retE((ret < 0));
return 0;
}
/*
* the routine to call when receving a response to @ping query.
*
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_ping_rsp(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_node_space* node_space = &route->node_space;
struct vroute_recr_space* recr_space = &route->recr_space;
vnodeInfo_relax nodei;
vtoken token;
int ret = 0;
ret = route->dec_ops->ping_rsp(ctxt, &token, (vnodeInfo*)&nodei);
retE((ret < 0));
retE((!recr_space->ops->check(recr_space, &token))); // skip vicious response.
ret = node_space->ops->add_node(node_space, (vnodeInfo*)&nodei, 1);
retE((ret < 0));
route->ops->inspect(route, &token, VROUTE_INSP_RCV_PING_RSP);
return 0;
}
/*
* the routine to call when receiving a @find_node query.
*
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_find_node(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_node_space* node_space = &route->node_space;
vnodeInfo_relax nodei;
struct varray closest;
vnodeId targetId;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
ret = route->dec_ops->find_node(ctxt, &token, &fromId, &targetId);
retE((ret < 0));
ret = node_space->ops->get_node(node_space, &targetId, (vnodeInfo*)&nodei);
retE((ret < 0));
if (ret == 1) { //means found
ret = route->dht_ops->find_node_rsp(route, conn, &token, (vnodeInfo*)&nodei);
retE((ret < 0));
return 0;
}
// otherwise, send @find_closest_nodes_rsp response instead.
varray_init(&closest, MAX_CAPC);
ret = node_space->ops->get_neighbors(node_space, &targetId, &closest, MAX_CAPC);
ret1E((ret < 0), varray_deinit(&closest));
if (ret == 0) { //means no closest nodes.
varray_deinit(&closest);
return 0;
}
ret = route->dht_ops->find_closest_nodes_rsp(route, conn, &token, &closest);
varray_zero(&closest, _aux_vnodeInfo_free, NULL);
varray_deinit(&closest);
retE((ret < 0));
return 0;
}
/*
* the routine to call when receiving a response to @find_node query.
*
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_find_node_rsp(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_node_space* node_space = &route->node_space;
struct vroute_recr_space* recr_space = &route->recr_space;
vnodeInfo_relax nodei;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
ret = route->dec_ops->find_node_rsp(ctxt, &token, &fromId, (vnodeInfo*)&nodei);
retE((ret < 0));
retE((!recr_space->ops->check(recr_space, &token)));
ret = node_space->ops->add_node(node_space, (vnodeInfo*)&nodei, 0);
retE((ret < 0));
route->ops->inspect(route, &token, VROUTE_INSP_RCV_FIND_NODE_RSP);
return 0;
}
/*
* the routine to call when receiving a @find_closest_nodes query.
*
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_find_closest_nodes(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_node_space* node_space = &route->node_space;
struct varray closest;
vnodeId targetId;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
ret = route->dec_ops->find_closest_nodes(ctxt, &token, &fromId, &targetId);
retE((ret < 0));
varray_init(&closest, MAX_CAPC);
ret = node_space->ops->get_neighbors(node_space, &targetId, &closest, MAX_CAPC);
ret1E((ret < 0), varray_deinit(&closest));
if (ret == 0) {
varray_deinit(&closest);
return 0; // not response if no closest nodes found.
}
ret = route->dht_ops->find_closest_nodes_rsp(route, conn, &token, &closest);
varray_zero(&closest, _aux_vnodeInfo_free, NULL);
varray_deinit(&closest);
retE((ret < 0));
return 0;
}
/*
* the routine to call when receiving a response to @find_closest_nodes
*
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_find_closest_nodes_rsp(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_recr_space* recr_space = &route->recr_space;
struct vroute_node_space* node_space = &route->node_space;
struct varray closest;
vnodeId fromId;
vtoken token;
int ret = 0;
int i = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
varray_init(&closest, MAX_CAPC);
ret = route->dec_ops->find_closest_nodes_rsp(ctxt, &token, &fromId, &closest);
ret1E((ret < 0), varray_deinit(&closest));
if (!recr_space->ops->check(recr_space, &token)) {
varray_zero(&closest, _aux_vnodeInfo_free, NULL);
varray_deinit(&closest);
return -1;
}
for (i = 0; i < varray_size(&closest); i++) {
node_space->ops->add_node(node_space, (vnodeInfo*)varray_get(&closest, i), 0);
}
varray_zero(&closest, _aux_vnodeInfo_free, NULL);
varray_deinit(&closest);
route->ops->inspect(route, &token, VROUTE_INSP_RCV_FIND_CLOSEST_NODES_RSP);
return 0;
}
/*
* the routine to call when receiving a @reflex query.
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_reflex(struct vroute* route, vnodeConn* conn, void* ctxt)
{
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
ret = route->dec_ops->reflex(ctxt, &token, &fromId);
retE((ret < 0));
ret = route->dht_ops->reflex_rsp(route, conn, &token, &conn->remote);
retE((ret < 0));
return 0;
}
/*
* the routine to call when receiving a response to @reflex
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_reflex_rsp(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_recr_space* recr_space = &route->recr_space;
struct vnode* node = route->node;
struct sockaddr_in reflexive_addr;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
ret = route->dec_ops->reflex_rsp(ctxt, &token, &fromId, &reflexive_addr);
retE((ret < 0));
retE((!recr_space->ops->check(recr_space, &token)));
ret = node->ops->reflex_addr(node, &conn->local, &reflexive_addr);
retE((ret < 0));
route->ops->inspect(route, &token, VROUTE_INSP_RCV_REFLEX_RSP);
return 0;
}
/*
* the routine to call when receiving a @probe query.
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_probe(struct vroute* route, vnodeConn* conn, void* ctxt)
{
vnodeId targetId;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
ret = route->dec_ops->probe(ctxt, &token, &fromId, &targetId);
retE((ret < 0));
retE((!vtoken_equal(&targetId, &route->myid)));
ret = route->dht_ops->probe_rsp(route, conn, &token);
retE((ret < 0));
return 0;
}
/*
* the routine to call when receiving a response to @probe query.
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_probe_rsp(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_recr_space* recr_space = &route->recr_space;
struct vroute_node_space* node_space = &route->node_space;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
ret = route->dec_ops->probe_rsp(ctxt, &token, &fromId);
retE((ret < 0));
retE((!recr_space->ops->check(recr_space, &token)));
ret = node_space->ops->adjust_connectivity(node_space, &fromId, conn);
retE((ret < 0));
route->ops->inspect(route, &token, VROUTE_INSP_RCV_PROBE_RSP);
return 0;
}
/*
* the routine to call when receiving a @post_service indication.
*
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_post_service(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_srvc_space* srvc_space = &route->srvc_space;
vsrvcInfo_relax srvci;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
ret = route->dec_ops->post_service(ctxt, &token, &fromId, (vsrvcInfo*)&srvci);
retE((ret < 0));
ret = srvc_space->ops->add_service(srvc_space, (vsrvcInfo*)&srvci);
retE((ret < 0));
route->ops->inspect(route, &token, VROUTE_INSP_RCV_POST_SERVICE);
return 0;
}
/*
* the routine to call when receiving a @find_service query.
*
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_find_service(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_srvc_space* srvc_space = &route->srvc_space;
vsrvcInfo_relax srvci;
vsrvcHash srvcHash;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(conn);
vassert(ctxt);
memset(&srvci, 0, sizeof(srvci));
srvci.capc = VSRVCINFO_MAX_ADDRS;
ret = route->dec_ops->find_service(ctxt, &token, &fromId, &srvcHash);
retE((ret < 0));
ret = srvc_space->ops->get_service(srvc_space, &srvcHash, (vsrvcInfo*)&srvci);
retE((ret < 0));
retS((ret == 0));
ret = route->dht_ops->find_service_rsp(route, conn, &token, (vsrvcInfo*)&srvci);
retE((ret < 0));
return 0;
}
/*
* the routine to call when receiving a response to @find_service.
*
* @route:
* @conn:
* @ctxt:
*/
static
int _vroute_cb_find_service_rsp(struct vroute* route, vnodeConn* conn, void* ctxt)
{
struct vroute_srvc_probe_helper* probe_helper = &route->probe_helper;
struct vroute_srvc_space* srvc_space = &route->srvc_space;
struct vroute_recr_space* recr_space = &route->recr_space;
struct vroute_node_space* node_space = &route->node_space;
vsrvcInfo_relax srvci;
vnodeId fromId;
vtoken token;
int ret = 0;
vassert(route);
vassert(ctxt);
vassert(conn);
ret = route->dec_ops->find_service_rsp(ctxt, &token, &fromId, (vsrvcInfo*)&srvci);
retE((ret < 0));
retE((!recr_space->ops->check(recr_space, &token)));
ret = srvc_space->ops->add_service(srvc_space, (vsrvcInfo*)&srvci);
retE((ret < 0));
//try to add info of node hosting that service.
ret = node_space->ops->probe_node(node_space, &srvci.hostid);
retE((ret < 0));
route->ops->inspect(route, &token, VROUTE_INSP_RCV_FIND_SERVICE_RSP);
probe_helper->ops->invoke(probe_helper, &srvci.hash, (vsrvcInfo*)&srvci);
return 0;
}
static
vroute_dht_cb_t route_cb_ops[] = {
_vroute_cb_ping,
_vroute_cb_ping_rsp,
_vroute_cb_find_node,
_vroute_cb_find_node_rsp,
_vroute_cb_find_closest_nodes,
_vroute_cb_find_closest_nodes_rsp,
_vroute_cb_reflex,
_vroute_cb_reflex_rsp,
_vroute_cb_probe,
_vroute_cb_probe_rsp,
_vroute_cb_post_service,
_vroute_cb_find_service,
_vroute_cb_find_service_rsp,
NULL
};
static
int _aux_route_msg_cb(void* cookie, struct vmsg_usr* mu)
{
struct vroute* route = (struct vroute*)cookie;
vnodeConn conn;
void* ctxt = NULL;
int ret = 0;
vassert(route);
vassert(mu);
ret = route->dec_ops->dec_begin(mu->data, mu->len, &ctxt);
retE((ret >= VDHT_UNKNOWN));
retE((ret < 0));
vlogD("received @%s", vdht_get_desc(ret));
vnodeConn_set(&conn, to_sockaddr_sin(mu->spec), to_sockaddr_sin(mu->addr));
vlock_enter(&route->lock);
ret = route->cb_ops[ret](route, &conn, ctxt);
vlock_leave(&route->lock);
route->dec_ops->dec_done(ctxt);
retE((ret < 0));
return 0;
}
static
int _aux_route_load_proto_caps(struct vconfig* cfg, uint32_t* props)
{
struct varray* tuple = NULL;
int i = 0;
vassert(cfg);
vassert(props);
tuple = cfg->ops->get_tuple_val(cfg, "dht.protocol");
if (!tuple) {
*props |= PROP_PING;
*props |= PROP_PING_R;
return 0;
}
for (i = 0; i < varray_size(tuple); i++) {
struct vcfg_item* item = NULL;
int dht_id = -1;
item = (struct vcfg_item*)varray_get(tuple, i);
retE((item->type != CFG_STR));
dht_id = vdht_get_dhtId_by_desc(item->val.s);
retE((dht_id < 0));
retE((dht_id >= VDHT_UNKNOWN));
*props |= (1 << dht_id);
}
return 0;
}
int vroute_init(struct vroute* route, struct vconfig* cfg, struct vhost* host, vnodeId* myid)
{
vassert(route);
vassert(host);
vassert(myid);
vtoken_copy(&route->myid, myid);
_aux_route_load_proto_caps(cfg, &route->props);
vlock_init(&route->lock);
vroute_node_space_init(&route->node_space, route, cfg, myid);
vroute_srvc_space_init(&route->srvc_space, cfg);
vroute_recr_space_init(&route->recr_space);
vroute_srvc_probe_helper_init(&route->probe_helper);
route->ops = &route_ops;
route->dht_ops = &route_dht_ops;
route->cb_ops = route_cb_ops;
route->enc_ops = &dht_enc_ops;
route->dec_ops = &dht_dec_ops;
route->cfg = cfg;
route->msger = &host->msger;
route->node = &host->node;
route->insp_cb = NULL;
route->insp_cookie = NULL;
route->msger->ops->add_cb(route->msger, route, _aux_route_msg_cb, VMSG_DHT);
return 0;
}
void vroute_deinit(struct vroute* route)
{
vassert(route);
vroute_srvc_probe_helper_deinit (&route->probe_helper);
vroute_recr_space_deinit(&route->recr_space);
vroute_node_space_deinit(&route->node_space);
vroute_srvc_space_deinit(&route->srvc_space);
vlock_deinit(&route->lock);
return ;
}
int
varnam_learn_from_file(varnam *handle,
const char *filepath,
vlearn_status *status,
void (*callback)(varnam *handle, const char *word, int status_code, void *object),
void *object)
{
int rc;
FILE *infile;
char line_buffer[10000];
strbuf *word;
varray *word_parts;
int confidence;
int parts;
infile = fopen(filepath, "r");
if (!infile) {
set_last_error (handle, "Couldn't open file '%s' for reading.\n", filepath);
return VARNAM_ERROR;
}
if (status != NULL)
{
status->total_words = 0;
status->failed = 0;
}
rc = vwt_optimize_for_huge_transaction(handle);
if (rc) {
fclose (infile);
return rc;
}
/* Learning from file will be mostly new words. Optimizing for that */
v_->_config_mostly_learning_new_words = 1;
varnam_log (handle, "Starting to learn from %s", filepath);
rc = vwt_start_changes (handle);
if (rc) {
vwt_turn_off_optimization_for_huge_transaction(handle);
fclose (infile);
return rc;
}
while (fgets(line_buffer, sizeof(line_buffer), infile))
{
reset_pool (handle);
word = get_pooled_string (handle);
strbuf_add (word, trimwhitespace (line_buffer));
word_parts = strbuf_split (word, handle, ' ');
parts = varray_length (word_parts);
if (parts > 0 && parts <= 2)
{
confidence = 1;
if (parts == 2) {
word = varray_get (word_parts, 1);
confidence = atoi (strbuf_to_s (word));
}
word = varray_get (word_parts, 0);
rc = varnam_learn_internal (handle, strbuf_to_s (word), confidence);
if (rc) {
if (status != NULL) status->failed++;
}
}
else {
rc = VARNAM_ERROR;
if (status != NULL) status->failed++;
}
if (status != NULL) status->total_words++;
if (callback != NULL) callback (handle, strbuf_to_s (word), rc, object);
}
varnam_log (handle, "Writing changes to disk");
rc = vwt_end_changes (handle);
if (rc) {
varnam_log (handle, "Writing changes to disk failed");
}
varnam_log (handle, "Ensuring file integrity");
rc = vwt_turn_off_optimization_for_huge_transaction(handle);
if (rc) {
varnam_log (handle, "Failed to check file integrity");
}
varnam_log (handle, "Compacting file");
rc = vwt_compact_file (handle);
if (rc) return rc;
fclose (infile);
return rc;
}
