/**
* This function builds an array containing the token numbers stored
* in the token list of the given state. As the token list is supposed
* to be sorted, the array will be sorted. The function frees the token
* list.
*/
static void token_list_2_token_array(OptimizedFst2State state,Abstract_allocator prv_alloc) {
int i;
struct opt_token* l;
struct opt_token* tmp;
if (state->number_of_tokens==0) {
/* Nothing to do if there is no token in the list */
return;
}
state->tokens=(int*)malloc_cb(sizeof(int)*state->number_of_tokens,prv_alloc);
if (state->tokens==NULL) {
fatal_alloc_error("token_list_2_token_array");
}
state->token_transitions=(Transition**)malloc_cb(sizeof(Transition*)*state->number_of_tokens,prv_alloc);
if (state->token_transitions==NULL) {
fatal_alloc_error("token_list_2_token_array");
}
i=0;
l=state->token_list;
while (l!=NULL) {
state->tokens[i]=l->token_number;
state->token_transitions[i]=l->transition;
i++;
tmp=l;
l=l->next;
/* We must NOT free 'tmp->transition' since it is referenced now
* in 'state->token_transitions[i]' */
free_cb(tmp,prv_alloc);
}
if (i!=state->number_of_tokens) {
fatal_error("Internal error in token_list_2_token_array\n");
}
state->token_list=NULL;
}
/**
* Allocates, initializes and returns a new token tree.
*/
struct fst2txt_token_tree* new_fst2txt_token_tree(Abstract_allocator prv_alloc) {
struct fst2txt_token_tree* t=(struct fst2txt_token_tree*)malloc_cb(sizeof(struct fst2txt_token_tree),prv_alloc);
if (t==NULL) {
fatal_alloc_error("new_fst2txt_token_tree");
}
t->hash=new_string_hash(DONT_USE_VALUES);
/* We set a small default capacity since there will be one structure of
* this kind for each state of the fst2 */
t->capacity=2;
t->size=0;
t->transition_array=(Transition**)malloc_cb(t->capacity*sizeof(Transition*), prv_alloc);
if (t->transition_array==NULL) {
fatal_alloc_error("new_fst2txt_token_tree");
}
return t;
}
/**
* Allocates, initializes and returns a new optimized state.
*/
static OptimizedFst2State new_optimized_state(Abstract_allocator prv_alloc) {
OptimizedFst2State state=(OptimizedFst2State)malloc_cb(sizeof(struct optimizedFst2State),prv_alloc);
if (state==NULL) {
fatal_alloc_error("new_optimized_state");
}
state->control=0;
state->graph_calls=NULL;
state->metas=NULL;
state->patterns=NULL;
state->compound_patterns=NULL;
state->token_list=NULL;
state->tokens=NULL;
state->token_transitions=NULL;
state->number_of_tokens=0;
state->input_variable_starts=NULL;
state->input_variable_ends=NULL;
state->output_variable_starts=NULL;
state->output_variable_ends=NULL;
state->contexts=NULL;
state->unoptimized_graph_calls=NULL;
state->unoptimized_metas=NULL;
state->unoptimized_input_variable_starts=NULL;
state->unoptimized_input_variable_ends=NULL;
state->unoptimized_output_variable_starts=NULL;
state->unoptimized_output_variable_ends=NULL;
return state;
}
/**
* Allocates, initializes and returns a pointer list cell.
*/
struct list_pointer* new_list_pointer(void* pointer,struct list_pointer* next,Abstract_allocator prv_alloc) {
struct list_pointer* p=(struct list_pointer*)malloc_cb(sizeof(struct list_pointer),prv_alloc);
if (p==NULL) {
fatal_alloc_error("new_list_pointer");
}
p->pointer=pointer;
p->next=next;
return p;
}
/**
* Allocates, initializes and returns a new int list element.
*/
struct list_int* new_list_int(int value,struct list_int* next,Abstract_allocator prv_alloc) {
struct list_int* l;
l=(struct list_int*)malloc_cb(sizeof(struct list_int),prv_alloc);
if (l==NULL) {
fatal_alloc_error("new_list_int");
}
l->n=value;
l->next=next;
return l;
}
/**
* Allocates, initializes and returns a new string_hash_tree_node.
*/
struct string_hash_tree_node* new_string_hash_tree_node(struct string_hash* s) {
struct string_hash_tree_node* node;
node=(struct string_hash_tree_node*)malloc_cb(sizeof(struct string_hash_tree_node),s->allocator_tree_node);
if (node==NULL) {
fatal_alloc_error("new_string_hash_tree_node");
}
node->value_index=NO_VALUE_INDEX;
node->trans=NULL;
return node;
}
/**
* Allocates, initializes and returns a new optimized token.
*/
static struct opt_token* new_opt_token(int token_number,Abstract_allocator prv_alloc) {
struct opt_token* t;
t=(struct opt_token*)malloc_cb(sizeof(struct opt_token),prv_alloc);
if (t==NULL) {
fatal_alloc_error("new_opt_token");
}
t->token_number=token_number;
t->transition=NULL;
t->next=NULL;
return t;
}
/**
* Allocates, initializes and returns a dictionary node transition.
*/
static inline struct dictionary_node_transition* new_dictionary_node_transition(Abstract_allocator prv_alloc) {
struct dictionary_node_transition* t=(struct dictionary_node_transition*)malloc_cb(sizeof(struct dictionary_node_transition),prv_alloc);
if (t==NULL) {
fatal_alloc_error("new_dictionary_node_transition");
}
t->letter='\0';
t->output=NULL;
t->node=NULL;
t->next=NULL;
return t;
}
/**
* Allocates, initializes and returns a new transition list element.
*/
static struct transition_list* new_transition_list(struct dictionary_node_transition* transition,
struct transition_list* next,Abstract_allocator prv_alloc) {
struct transition_list* t;
t=(struct transition_list*)malloc_cb(sizeof(struct transition_list),prv_alloc);
if (t==NULL) {
fatal_alloc_error("new_transition_list");
}
t->transition=transition;
t->next=next;
return t;
}
/**
* Allocates, initializes and returns a new string_hash_tree_transition.
*/
struct string_hash_tree_transition* new_string_hash_tree_transition(struct string_hash* s) {
struct string_hash_tree_transition* transition;
transition=(struct string_hash_tree_transition*)malloc_cb(sizeof(struct string_hash_tree_transition),s->allocator_tree_transition);
if (transition==NULL) {
fatal_alloc_error("new_string_hash_tree_transition");
}
transition->letter='\0';
transition->node=NULL;
transition->next=NULL;
return transition;
}
/**
* Creates, initializes and returns a transition tagged by an integer.
*/
Transition* new_Transition(int tag_number,int state_number,Transition* next,Abstract_allocator prv_alloc) {
Transition* transition;
transition=(Transition*)malloc_cb(sizeof(Transition),prv_alloc);
if (transition==NULL) {
fatal_alloc_error("new_Transition");
}
transition->tag_number=tag_number;
transition->state_number=state_number;
transition->next=next;
return transition;
}
/**
* Allocates, initializes and returns a new optimized graph call.
*/
static struct opt_graph_call* new_opt_graph_call(int graph_number,Abstract_allocator prv_alloc) {
struct opt_graph_call* g;
g=(struct opt_graph_call*)malloc_cb(sizeof(struct opt_graph_call),prv_alloc);
if (g==NULL) {
fatal_alloc_error("new_opt_graph_call");
}
g->graph_number=graph_number;
g->transition=NULL;
g->next=NULL;
return g;
}
/**
* The same than above, except that it does not duplicate the given symbol.
*/
Transition* new_Transition_no_dup(symbol_t* label,int state_number,Transition* next,Abstract_allocator prv_alloc) {
Transition* transition;
transition=(Transition*)malloc_cb(sizeof(Transition),prv_alloc);
if (transition==NULL) {
fatal_alloc_error("new_Transition");
}
transition->label=label;
transition->state_number=state_number;
transition->next=next;
return transition;
}
/**
* Builds, initializes and returns a new LocateCache.
*/
LocateCache new_LocateCache(int token,struct match_list* matches,Abstract_allocator prv_alloc) {
LocateCache c=(LocateCache)malloc_cb(sizeof(struct locate_cache),prv_alloc);
if (c==NULL) {
fatal_alloc_error("new_LocateCache");
}
c->left=NULL;
c->middle=NULL;
c->right=NULL;
c->token=token;
c->matches=matches;
return c;
}
/**
* Allocates, initializes and returns a new opt_pattern.
*/
struct opt_pattern* new_opt_pattern(int pattern_number,int negation,Abstract_allocator prv_alloc) {
struct opt_pattern* p;
p=(struct opt_pattern*)malloc_cb(sizeof(struct opt_pattern),prv_alloc);
if (p==NULL) {
fatal_alloc_error("new_opt_pattern");
}
p->pattern_number=pattern_number;
p->negation=(char)negation;
p->transition=NULL;
p->next=NULL;
return p;
}
/**
* Allocates, initializes and returns a dictionary node.
*/
struct dictionary_node* new_dictionary_node(Abstract_allocator prv_alloc) {
struct dictionary_node* a=(struct dictionary_node*)malloc_cb(sizeof(struct dictionary_node),prv_alloc);
if (a==NULL) {
fatal_alloc_error("new_dictionary_node");
}
a->single_INF_code_list=NULL;
a->offset=-1;
a->trans=NULL;
a->incoming=0;
a->n_trans=0;
a->INF_code=0;
return a;
}
/**
* Allocates, initializes and returns a new optimized variable.
*/
static struct opt_variable* new_opt_variable(int variable_number,Transition* transition,
Abstract_allocator prv_alloc) {
struct opt_variable* v;
v=(struct opt_variable*)malloc_cb(sizeof(struct opt_variable),prv_alloc);
if (v==NULL) {
fatal_alloc_error("new_opt_variable");
}
v->variable_number=variable_number;
v->transition=NULL;
add_transition_if_not_present(&(v->transition),transition->tag_number,transition->state_number,prv_alloc);
v->next=NULL;
return v;
}
/**
* Allocates, initializes and returns a new optimized meta.
*/
static struct opt_meta* new_opt_meta(enum meta_symbol meta,int negation,Abstract_allocator prv_alloc) {
struct opt_meta* m;
m=(struct opt_meta*)malloc_cb(sizeof(struct opt_meta),prv_alloc);
if (m==NULL) {
fatal_alloc_error("new_opt_meta");
}
m->meta=meta;
m->negation=(char)negation;
m->transition=NULL;
m->next=NULL;
m->morphological_mode_ends=NULL;
return m;
}
/**
* Allocates, initializes and returns a new pattern.
*/
struct pattern* new_pattern(Abstract_allocator prv_alloc) {
struct pattern* p;
p=(struct pattern*)malloc_cb(sizeof(struct pattern),prv_alloc);
if (p==NULL) {
fatal_alloc_error("new_pattern");
}
p->inflected=NULL;
p->lemma=NULL;
p->grammatical_codes=NULL;
p->inflectional_codes=NULL;
p->forbidden_codes=NULL;
p->type=UNDEFINED_PATTERN;
return p;
}
Transition* clone_transition(const Transition* t,symbol_t*(*clone_elag_symbol)(const symbol_t*),Abstract_allocator prv_alloc) {
Transition* transition;
transition=(Transition*)malloc_cb(sizeof(Transition),prv_alloc);
if (transition==NULL) {
fatal_alloc_error("clone_transition");
}
if (clone_elag_symbol==NULL) {
memcpy(transition,t,sizeof(Transition));
} else {
transition->label=clone_elag_symbol(t->label);
transition->state_number=t->state_number;
transition->next=t->next;
}
return transition;
}
/**
* Returns a clone of the pattern.
*/
struct pattern* clone(const struct pattern* src,Abstract_allocator prv_alloc) {
struct pattern* dst;
if (src == NULL)
return NULL;
dst=(struct pattern*)malloc_cb(sizeof(struct pattern),prv_alloc);
if (dst==NULL) {
fatal_error("Not enough memory in new_pattern_ByCopy\n");
}
dst->inflected=u_strdup(src->inflected,prv_alloc);
dst->lemma=u_strdup(src->lemma,prv_alloc);
dst->grammatical_codes=clone(src->grammatical_codes,prv_alloc);
dst->inflectional_codes=clone(src->inflectional_codes,prv_alloc);
dst->forbidden_codes=clone(src->forbidden_codes,prv_alloc);
dst->type=src->type;
return dst;
}
/**
* Allocates, initializes and returns an integer array that contains
* the elements of the given list. '*size' is set to the size of this
* array. Note that passing an empty list will return NULL.
*/
int* dump(struct list_int* list,int *size,Abstract_allocator prv_alloc) {
*size=0;
if (list==NULL) return NULL;
struct list_int* tmp=list;
/* We count the number of elements */
while (tmp!=NULL) {
(*size)++;
tmp=tmp->next;
}
int* result=(int*)malloc_cb((*size)*sizeof(int),prv_alloc);
if (result==NULL) {
fatal_alloc_error("dump");
}
tmp=list;
for (int i=0;i<(*size);i++) {
result[i]=tmp->n;
tmp=tmp->next;
}
return result;
}
/**
* This function takes a fst2 and returns an array containing the corresponding
* optimized states.
*/
OptimizedFst2State* build_optimized_fst2_states(Variables* v,OutputVariables* output,Fst2* fst2,
Abstract_allocator prv_alloc) {
OptimizedFst2State* optimized_states=(OptimizedFst2State*)malloc_cb(fst2->number_of_states*sizeof(OptimizedFst2State),prv_alloc);
if (optimized_states==NULL) {
fatal_alloc_error("build_optimized_fst2_states");
}
int num_current_graph=1;
int pos_in_current_graph=0;
for (int i=0;i<fst2->number_of_states;i++) {
optimized_states[i]=optimize_state(v,output,fst2,fst2->states[i],fst2->tags,prv_alloc);
optimized_states[i]->graph_number=num_current_graph;
optimized_states[i]->pos_transition_in_fst2=i;
optimized_states[i]->pos_transition_in_graph=pos_in_current_graph++;
if (pos_in_current_graph >= *((fst2->number_of_states_per_graphs)+num_current_graph))
{
num_current_graph++;
pos_in_current_graph=0;
}
}
#ifdef AGGRESSIVE_OPTIMIZATION
int n_graphs_emptied;
do {
n_graphs_emptied=0;
for (int i=1;i<=fst2->number_of_graphs;i++) {
n_graphs_emptied+=remove_useless_lexical_transitions(fst2,i,optimized_states,prv_alloc);
}
} while (n_graphs_emptied!=0);
/* Finally, we convert token lists to sorted array suitable for binary search */
for (int i=0;i<fst2->number_of_states;i++) {
token_list_2_token_array(optimized_states[i],prv_alloc);
}
#endif // AGGRESSIVE_OPTIMIZATION
return optimized_states;
}
/**
* Allocates, initializes and returns a new match list element.
*/
struct match_list* new_match(int start,int end,int start_char,int end_char,
int start_letter,int end_letter,unichar* output,
int weight,struct match_list* next,Abstract_allocator prv_alloc) {
struct match_list *l;
l=(struct match_list*)malloc_cb(sizeof(struct match_list),prv_alloc);
if (l==NULL) {
fatal_alloc_error("new_match");
}
l->m.start_pos_in_token=start;
l->m.end_pos_in_token=end;
l->weight=weight;
if (output==NULL) {
l->output=NULL;
} else {
l->output=u_strdup(output,prv_alloc);
}
l->m.start_pos_in_char=start_char;
l->m.end_pos_in_char=end_char;
l->m.start_pos_in_letter=start_letter;
l->m.end_pos_in_letter=end_letter;
l->next=next;
return l;
}
connector_handle_t connector_init(connector_callback_t const callback)
{
connector_data_t * connector_handle = NULL;
connector_status_t status;
#if (defined CONNECTOR_SW_DESCRIPTION)
connector_debug_printf("Cloud Connector v%s %s\n", CONNECTOR_SW_VERSION, CONNECTOR_SW_DESCRIPTION);
#else
connector_debug_printf("Cloud Connector v%s\n", CONNECTOR_SW_VERSION);
#endif
{
void * handle;
#if (defined CONNECTOR_NO_MALLOC)
status = malloc_data_buffer(NULL, sizeof *connector_handle, named_buffer_id(connector_data), &handle);
#else
status = malloc_cb(callback, sizeof *connector_handle, &handle);
#endif
COND_ELSE_GOTO(status == connector_working, done);
memset(handle, 0x00, sizeof *connector_handle); /* Init structure, all pointers to NULL */
connector_handle = handle;
}
connector_handle->callback = callback;
status = manage_device_id(connector_handle);
COND_ELSE_GOTO(status == connector_working, error);
/* make a copy of the cloud url */
#if (defined CONNECTOR_TRANSPORT_TCP) || (defined CONNECTOR_TRANSPORT_UDP)
#if (defined CONNECTOR_CLOUD_URL)
{
static char const connector_device_cloud_url[]= CONNECTOR_CLOUD_URL;
connector_handle->device_cloud_url = (char *)connector_device_cloud_url;
connector_handle->device_cloud_url_length = sizeof connector_device_cloud_url -1;
}
#else
status = get_config_device_cloud_url(connector_handle);
COND_ELSE_GOTO(status == connector_working, error);
#endif
#endif /* (defined CONNECTOR_TRANSPORT_TCP) || (defined CONNECTOR_TRANSPORT_UDP) */
/* make a copy of the cloud phone */
#if (defined CONNECTOR_TRANSPORT_SMS)
#if (defined CONNECTOR_CLOUD_PHONE)
{
static char const connector_device_cloud_phone[]= CONNECTOR_CLOUD_PHONE;
connector_handle->device_cloud_phone = (char *)connector_device_cloud_phone;
connector_handle->device_cloud_phone_length = sizeof connector_device_cloud_phone -1;
}
#else
status = get_config_device_cloud_phone(connector_handle);
COND_ELSE_GOTO(status == connector_working, error);
#endif
#if (defined CONNECTOR_CLOUD_SERVICE_ID)
{
static char const connector_device_cloud_service_id[]= CONNECTOR_CLOUD_SERVICE_ID;
connector_handle->device_cloud_service_id = (char *)connector_device_cloud_service_id;
connector_handle->device_cloud_service_id_length = sizeof connector_device_cloud_service_id -1;
}
#else
status = get_config_device_cloud_service_id(connector_handle);
COND_ELSE_GOTO(status == connector_working, error);
#endif
#endif /* (defined CONNECTOR_TRANSPORT_SMS) */
#if (defined CONNECTOR_TRANSPORT_TCP)
status = connector_edp_init(connector_handle);
COND_ELSE_GOTO(status == connector_working, error);
#endif
#if (defined CONNECTOR_TRANSPORT_UDP) || (defined CONNECTOR_TRANSPORT_SMS)
status = connector_sm_init(connector_handle);
COND_ELSE_GOTO(status == connector_working, error);
#endif
#if (defined CONNECTOR_TRANSPORT_COUNT > 1)
connector_handle->first_running_network = (connector_network_type_t) 0;
#endif
connector_handle->signature = connector_signature;
goto done;
error:
free_data_buffer(connector_handle, named_buffer_id(connector_data), connector_handle);
connector_handle = NULL;
done:
return connector_handle;
}
int locate_pattern(const char* text_cod,const char* tokens,const char* fst2_name,const char* dlf,const char* dlc,const char* err,
const char* alphabet,MatchPolicy match_policy,OutputPolicy output_policy,
Encoding encoding_output,int bom_output,int mask_encoding_compatibility_input,
const char* dynamicDir,TokenizationPolicy tokenization_policy,
SpacePolicy space_policy,int search_limit,const char* morpho_dic_list,
AmbiguousOutputPolicy ambiguous_output_policy,
VariableErrorPolicy variable_error_policy,int protect_dic_chars,
int is_korean,int max_count_call,int max_count_call_warning,
char* arabic_rules,int tilde_negation_operator,int useLocateCache,int allow_trace) {
U_FILE* out;
U_FILE* info;
struct locate_parameters* p=new_locate_parameters();
p->text_cod=af_open_mapfile(text_cod,MAPFILE_OPTION_READ,0);
p->buffer=(int*)af_get_mapfile_pointer(p->text_cod);
long text_size=(long)af_get_mapfile_size(p->text_cod)/sizeof(int);
p->buffer_size=(int)text_size;
p->tilde_negation_operator=tilde_negation_operator;
p->useLocateCache=useLocateCache;
if (max_count_call == -1) {
max_count_call = (int)text_size;
}
if (max_count_call_warning == -1) {
max_count_call_warning = (int)text_size;
}
p->match_policy=match_policy;
p->tokenization_policy=tokenization_policy;
p->space_policy=space_policy;
p->output_policy=output_policy;
p->search_limit=search_limit;
p->ambiguous_output_policy=ambiguous_output_policy;
p->variable_error_policy=variable_error_policy;
p->protect_dic_chars=protect_dic_chars;
p->mask_encoding_compatibility_input = mask_encoding_compatibility_input;
p->max_count_call = max_count_call;
p->max_count_call_warning = max_count_call_warning;
p->token_filename = tokens;
char concord[FILENAME_MAX];
char concord_info[FILENAME_MAX];
strcpy(concord,dynamicDir);
strcat(concord,"concord.ind");
strcpy(concord_info,dynamicDir);
strcat(concord_info,"concord.n");
char morpho_bin[FILENAME_MAX];
strcpy(morpho_bin,dynamicDir);
strcat(morpho_bin,"morpho.bin");
if (arabic_rules!=NULL && arabic_rules[0]!='\0') {
load_arabic_typo_rules(arabic_rules,&(p->arabic));
}
out=u_fopen_versatile_encoding(encoding_output,bom_output,mask_encoding_compatibility_input,concord,U_WRITE);
if (out==NULL) {
error("Cannot write %s\n",concord);
af_release_mapfile_pointer(p->text_cod,p->buffer);
af_close_mapfile(p->text_cod);
free_stack_unichar(p->stack);
free_locate_parameters(p);
u_fclose(out);
return 0;
}
info=u_fopen_versatile_encoding(encoding_output,bom_output,mask_encoding_compatibility_input,concord_info,U_WRITE);
if (info==NULL) {
error("Cannot write %s\n",concord_info);
}
switch(output_policy) {
case IGNORE_OUTPUTS:
u_fprintf(out,"#I\n");
break;
case MERGE_OUTPUTS:
u_fprintf(out,"#M\n");
break;
case REPLACE_OUTPUTS:
u_fprintf(out,"#R\n");
break;
}
if (alphabet!=NULL && alphabet[0]!='\0') {
u_printf("Loading alphabet...\n");
p->alphabet=load_alphabet(alphabet,is_korean);
if (p->alphabet==NULL) {
error("Cannot load alphabet file %s\n",alphabet);
af_release_mapfile_pointer(p->text_cod,p->buffer);
af_close_mapfile(p->text_cod);
free_stack_unichar(p->stack);
free_locate_parameters(p);
if (info!=NULL) u_fclose(info);
u_fclose(out);
return 0;
}
}
struct string_hash* semantic_codes=new_string_hash();
extract_semantic_codes(dlf,semantic_codes);
extract_semantic_codes(dlc,semantic_codes);
if (is_cancelling_requested() != 0) {
error("user cancel request.\n");
free_alphabet(p->alphabet);
free_string_hash(semantic_codes);
af_release_mapfile_pointer(p->text_cod,p->buffer);
af_close_mapfile(p->text_cod);
free_stack_unichar(p->stack);
free_locate_parameters(p);
if (info!=NULL) u_fclose(info);
u_fclose(out);
return 0;
}
u_printf("Loading fst2...\n");
struct FST2_free_info fst2load_free;
Fst2* fst2load=load_abstract_fst2(fst2_name,1,&fst2load_free);
if (fst2load==NULL) {
error("Cannot load grammar %s\n",fst2_name);
free_alphabet(p->alphabet);
free_string_hash(semantic_codes);
af_release_mapfile_pointer(p->text_cod,p->buffer);
af_close_mapfile(p->text_cod);
free_stack_unichar(p->stack);
free_locate_parameters(p);
if (info!=NULL) u_fclose(info);
u_fclose(out);
return 0;
}
Abstract_allocator locate_abstract_allocator=create_abstract_allocator("locate_pattern",AllocatorCreationFlagAutoFreePrefered);
p->fst2=new_Fst2_clone(fst2load,locate_abstract_allocator);
free_abstract_Fst2(fst2load,&fst2load_free);
if (is_cancelling_requested() != 0) {
error("User cancel request..\n");
free_alphabet(p->alphabet);
free_string_hash(semantic_codes);
free_Fst2(p->fst2,locate_abstract_allocator);
close_abstract_allocator(locate_abstract_allocator);
af_release_mapfile_pointer(p->text_cod,p->buffer);
af_close_mapfile(p->text_cod);
free_stack_unichar(p->stack);
free_locate_parameters(p);
if (info!=NULL) u_fclose(info);
u_fclose(out);
return 0;
}
p->tags=p->fst2->tags;
#ifdef TRE_WCHAR
p->filters=new_FilterSet(p->fst2,p->alphabet);
if (p->filters==NULL) {
error("Cannot compile filter(s)\n");
free_alphabet(p->alphabet);
free_string_hash(semantic_codes);
free_Fst2(p->fst2,locate_abstract_allocator);
close_abstract_allocator(locate_abstract_allocator);
free_stack_unichar(p->stack);
free_locate_parameters(p);
af_release_mapfile_pointer(p->text_cod,p->buffer);
af_close_mapfile(p->text_cod);
if (info!=NULL) u_fclose(info);
u_fclose(out);
return 0;
}
#endif
u_printf("Loading token list...\n");
int n_text_tokens=0;
p->tokens=load_text_tokens_hash(tokens,mask_encoding_compatibility_input,&(p->SENTENCE),&(p->STOP),&n_text_tokens);
if (p->tokens==NULL) {
error("Cannot load token list %s\n",tokens);
free_alphabet(p->alphabet);
free_string_hash(semantic_codes);
free_Fst2(p->fst2,locate_abstract_allocator);
close_abstract_allocator(locate_abstract_allocator);
free_locate_parameters(p);
af_release_mapfile_pointer(p->text_cod,p->buffer);
af_close_mapfile(p->text_cod);
if (info!=NULL) u_fclose(info);
u_fclose(out);
return 0;
}
Abstract_allocator locate_work_abstract_allocator = locate_abstract_allocator;
p->match_cache=(LocateCache*)malloc_cb(p->tokens->size * sizeof(LocateCache),locate_work_abstract_allocator);
memset(p->match_cache,0,p->tokens->size * sizeof(LocateCache));
if (p->match_cache==NULL) {
fatal_alloc_error("locate_pattern");
}
#ifdef TRE_WCHAR
p->filter_match_index=new_FilterMatchIndex(p->filters,p->tokens);
if (p->filter_match_index==NULL) {
error("Cannot optimize filter(s)\n");
free_alphabet(p->alphabet);
free_string_hash(semantic_codes);
free_string_hash(p->tokens);
close_abstract_allocator(locate_abstract_allocator);
free_locate_parameters(p);
af_release_mapfile_pointer(p->text_cod,p->buffer);
af_close_mapfile(p->text_cod);
if (info!=NULL) u_fclose(info);
u_fclose(out);
return 0;
}
#endif
if (allow_trace!=0) {
open_locate_trace(p,&p->fnc_locate_trace_step,&p->private_param_locate_trace);
}
extract_semantic_codes_from_tokens(p->tokens,semantic_codes,locate_abstract_allocator);
u_printf("Loading morphological dictionaries...\n");
load_morphological_dictionaries(morpho_dic_list,p,morpho_bin);
extract_semantic_codes_from_morpho_dics(p->morpho_dic_inf,p->n_morpho_dics,semantic_codes,locate_abstract_allocator);
p->token_control=(unsigned char*)malloc(n_text_tokens*sizeof(unsigned char));
if (p->token_control==NULL) {
fatal_alloc_error("locate_pattern");
}
p->matching_patterns=(struct bit_array**)malloc(n_text_tokens*sizeof(struct bit_array*));
if (p->matching_patterns==NULL) {
fatal_alloc_error("locate_pattern");
}
for (int i=0; i<n_text_tokens; i++) {
p->token_control[i]=0;
p->matching_patterns[i]=NULL;
}
compute_token_controls(p->alphabet,err,p);
int number_of_patterns,is_DIC,is_CDIC,is_SDIC;
p->pattern_tree_root=new_pattern_node(locate_abstract_allocator);
u_printf("Computing fst2 tags...\n");
process_tags(&number_of_patterns,semantic_codes,&is_DIC,&is_CDIC,&is_SDIC,p,locate_abstract_allocator);
p->current_compound_pattern=number_of_patterns;
p->DLC_tree=new_DLC_tree(p->tokens->size);
struct lemma_node* root=new_lemma_node();
u_printf("Loading dlf...\n");
load_dic_for_locate(dlf,mask_encoding_compatibility_input,p->alphabet,number_of_patterns,is_DIC,is_CDIC,root,p);
u_printf("Loading dlc...\n");
load_dic_for_locate(dlc,mask_encoding_compatibility_input,p->alphabet,number_of_patterns,is_DIC,is_CDIC,root,p);
/* We look if tag tokens like "{today,.ADV}" verify some patterns */
check_patterns_for_tag_tokens(p->alphabet,number_of_patterns,root,p,locate_abstract_allocator);
u_printf("Optimizing fst2 pattern tags...\n");
optimize_pattern_tags(p->alphabet,root,p,locate_abstract_allocator);
u_printf("Optimizing compound word dictionary...\n");
optimize_DLC(p->DLC_tree);
free_string_hash(semantic_codes);
int nb_input_variable=0;
p->input_variables=new_Variables(p->fst2->input_variables,&nb_input_variable);
p->output_variables=new_OutputVariables(p->fst2->output_variables,&p->nb_output_variables);
Abstract_allocator locate_recycle_abstract_allocator=NULL;
locate_recycle_abstract_allocator=create_abstract_allocator("locate_pattern_recycle",
AllocatorFreeOnlyAtAllocatorDelete|AllocatorTipOftenRecycledObject,
get_prefered_allocator_item_size_for_nb_variable(nb_input_variable));
u_printf("Optimizing fst2...\n");
p->optimized_states=build_optimized_fst2_states(p->input_variables,p->output_variables,p->fst2,locate_abstract_allocator);
if (is_korean) {
p->korean=new Korean(p->alphabet);
p->jamo_tags=create_jamo_tags(p->korean,p->tokens);
}
p->failfast=new_bit_array(n_text_tokens,ONE_BIT);
u_printf("Working...\n");
p->prv_alloc=locate_work_abstract_allocator;
p->prv_alloc_recycle=locate_recycle_abstract_allocator;
launch_locate(out,text_size,info,p);
if (allow_trace!=0) {
close_locate_trace(p,p->fnc_locate_trace_step,p->private_param_locate_trace);
}
free_bit_array(p->failfast);
free_Variables(p->input_variables);
free_OutputVariables(p->output_variables);
af_release_mapfile_pointer(p->text_cod,p->buffer);
af_close_mapfile(p->text_cod);
if (info!=NULL) u_fclose(info);
u_fclose(out);
if (p->match_cache!=NULL) {
for (int i=0; i<p->tokens->size; i++) {
free_LocateCache(p->match_cache[i],locate_work_abstract_allocator);
}
free_cb(p->match_cache,locate_work_abstract_allocator);
}
int free_abstract_allocator_item=(get_allocator_cb_flag(locate_abstract_allocator) & AllocatorGetFlagAutoFreePresent) ? 0 : 1;
if (free_abstract_allocator_item) {
free_optimized_states(p->optimized_states,p->fst2->number_of_states,locate_abstract_allocator);
}
free_stack_unichar(p->stack);
/** Too long to free the DLC tree if it is big
* free_DLC_tree(p->DLC_tree);
*/
if (free_abstract_allocator_item) {
free_pattern_node(p->pattern_tree_root,locate_abstract_allocator);
free_Fst2(p->fst2,locate_abstract_allocator);
free_list_int(p->tag_token_list,locate_abstract_allocator);
}
close_abstract_allocator(locate_abstract_allocator);
close_abstract_allocator(locate_recycle_abstract_allocator);
locate_recycle_abstract_allocator=locate_abstract_allocator=NULL;
/* We don't free 'parameters->tags' because it was just a link on 'parameters->fst2->tags' */
free_alphabet(p->alphabet);
if (p->korean!=NULL) {
delete p->korean;
}
if (p->jamo_tags!=NULL) {
/* jamo tags must be freed before tokens, because we need to know how
* many jamo tags there are, and this number is the number of tokens */
for (int i=0; i<p->tokens->size; i++) {
free(p->jamo_tags[i]);
}
free(p->jamo_tags);
}
free_string_hash(p->tokens);
free_lemma_node(root);
free(p->token_control);
for (int i=0; i<n_text_tokens; i++) {
free_bit_array(p->matching_patterns[i]);
}
free(p->matching_patterns);
#ifdef TRE_WCHAR
free_FilterSet(p->filters);
free_FilterMatchIndex(p->filter_match_index);
#endif
for (int i=0; i<p->n_morpho_dics; i++) {
free_abstract_INF(p->morpho_dic_inf[i],&(p->morpho_dic_inf_free[i]));
free_abstract_BIN(p->morpho_dic_bin[i],&(p->morpho_dic_bin_free[i]));
}
free(p->morpho_dic_inf);
free(p->morpho_dic_inf_free);
free(p->morpho_dic_bin);
free(p->morpho_dic_bin_free);
#if (defined(UNITEX_LIBRARY) || defined(UNITEX_RELEASE_MEMORY_AT_EXIT))
free_DLC_tree(p->DLC_tree);
#endif
free_locate_parameters(p);
u_printf("Done.\n");
return 1;
}
