/**
 * This function builds an array containing the token numbers stored
 * in the token list of the given state. As the token list is supposed
 * to be sorted, the array will be sorted. The function frees the token
 * list.
 */
static void token_list_2_token_array(OptimizedFst2State state,Abstract_allocator prv_alloc) {
int i;
struct opt_token* l;
struct opt_token* tmp;
if (state->number_of_tokens==0) {
   /* Nothing to do if there is no token in the list */
   return;
}
state->tokens=(int*)malloc_cb(sizeof(int)*state->number_of_tokens,prv_alloc);
if (state->tokens==NULL) {
   fatal_alloc_error("token_list_2_token_array");
}
state->token_transitions=(Transition**)malloc_cb(sizeof(Transition*)*state->number_of_tokens,prv_alloc);
if (state->token_transitions==NULL) {
   fatal_alloc_error("token_list_2_token_array");
}
i=0;
l=state->token_list;
while (l!=NULL) {
   state->tokens[i]=l->token_number;
   state->token_transitions[i]=l->transition;
   i++;
   tmp=l;
   l=l->next;
   /* We must NOT free 'tmp->transition' since it is referenced now
    * in 'state->token_transitions[i]' */
   free_cb(tmp,prv_alloc);
}
if (i!=state->number_of_tokens) {
   fatal_error("Internal error in token_list_2_token_array\n");
}
state->token_list=NULL;
}
/**
 * Allocates, initializes and returns a new token tree.
 */
struct fst2txt_token_tree* new_fst2txt_token_tree(Abstract_allocator prv_alloc) {
struct fst2txt_token_tree* t=(struct fst2txt_token_tree*)malloc_cb(sizeof(struct fst2txt_token_tree),prv_alloc);
if (t==NULL) {
   fatal_alloc_error("new_fst2txt_token_tree");
}
t->hash=new_string_hash(DONT_USE_VALUES);
/* We set a small default capacity since there will be one structure of
 * this kind for each state of the fst2 */
t->capacity=2;
t->size=0;
t->transition_array=(Transition**)malloc_cb(t->capacity*sizeof(Transition*), prv_alloc);
if (t->transition_array==NULL) {
   fatal_alloc_error("new_fst2txt_token_tree");
}
return t;
}
/**
 * Allocates, initializes and returns a new optimized state.
 */
static OptimizedFst2State new_optimized_state(Abstract_allocator prv_alloc) {
OptimizedFst2State state=(OptimizedFst2State)malloc_cb(sizeof(struct optimizedFst2State),prv_alloc);
if (state==NULL) {
   fatal_alloc_error("new_optimized_state");
}
state->control=0;
state->graph_calls=NULL;
state->metas=NULL;
state->patterns=NULL;
state->compound_patterns=NULL;
state->token_list=NULL;
state->tokens=NULL;
state->token_transitions=NULL;
state->number_of_tokens=0;
state->input_variable_starts=NULL;
state->input_variable_ends=NULL;
state->output_variable_starts=NULL;
state->output_variable_ends=NULL;
state->contexts=NULL;
state->unoptimized_graph_calls=NULL;
state->unoptimized_metas=NULL;
state->unoptimized_input_variable_starts=NULL;
state->unoptimized_input_variable_ends=NULL;
state->unoptimized_output_variable_starts=NULL;
state->unoptimized_output_variable_ends=NULL;
return state;
}
Esempio n. 4
0
/**
 * Allocates, initializes and returns a pointer list cell.
 */
struct list_pointer* new_list_pointer(void* pointer,struct list_pointer* next,Abstract_allocator prv_alloc) {
struct list_pointer* p=(struct list_pointer*)malloc_cb(sizeof(struct list_pointer),prv_alloc);
if (p==NULL) {
   fatal_alloc_error("new_list_pointer");
}
p->pointer=pointer;
p->next=next;
return p;
}
Esempio n. 5
0
/**
 * Allocates, initializes and returns a new int list element.
 */
struct list_int* new_list_int(int value,struct list_int* next,Abstract_allocator prv_alloc) {
struct list_int* l;
l=(struct list_int*)malloc_cb(sizeof(struct list_int),prv_alloc);
if (l==NULL) {
   fatal_alloc_error("new_list_int");
}
l->n=value;
l->next=next;
return l;
}
Esempio n. 6
0
/**
 * Allocates, initializes and returns a new string_hash_tree_node.
 */
struct string_hash_tree_node* new_string_hash_tree_node(struct string_hash* s) {
struct string_hash_tree_node* node;
node=(struct string_hash_tree_node*)malloc_cb(sizeof(struct string_hash_tree_node),s->allocator_tree_node);
if (node==NULL) {
   fatal_alloc_error("new_string_hash_tree_node");
}
node->value_index=NO_VALUE_INDEX;
node->trans=NULL;
return node;
}
/**
 * Allocates, initializes and returns a new optimized token.
 */
static struct opt_token* new_opt_token(int token_number,Abstract_allocator prv_alloc) {
struct opt_token* t;
t=(struct opt_token*)malloc_cb(sizeof(struct opt_token),prv_alloc);
if (t==NULL) {
   fatal_alloc_error("new_opt_token");
}
t->token_number=token_number;
t->transition=NULL;
t->next=NULL;
return t;
}
/**
 * Allocates, initializes and returns a dictionary node transition.
 */
static inline struct dictionary_node_transition* new_dictionary_node_transition(Abstract_allocator prv_alloc) {
struct dictionary_node_transition* t=(struct dictionary_node_transition*)malloc_cb(sizeof(struct dictionary_node_transition),prv_alloc);
if (t==NULL) {
	fatal_alloc_error("new_dictionary_node_transition");
}
t->letter='\0';
t->output=NULL;
t->node=NULL;
t->next=NULL;
return t;
}
/**
 * Allocates, initializes and returns a new transition list element.
 */
static struct transition_list* new_transition_list(struct dictionary_node_transition* transition,
                                      struct transition_list* next,Abstract_allocator prv_alloc) {
struct transition_list* t;
t=(struct transition_list*)malloc_cb(sizeof(struct transition_list),prv_alloc);
if (t==NULL) {
   fatal_alloc_error("new_transition_list");
}
t->transition=transition;
t->next=next;
return t;
}
Esempio n. 10
0
/**
 * Allocates, initializes and returns a new string_hash_tree_transition.
 */
struct string_hash_tree_transition* new_string_hash_tree_transition(struct string_hash* s) {
struct string_hash_tree_transition* transition;
transition=(struct string_hash_tree_transition*)malloc_cb(sizeof(struct string_hash_tree_transition),s->allocator_tree_transition);
if (transition==NULL) {
   fatal_alloc_error("new_string_hash_tree_transition");
}
transition->letter='\0';
transition->node=NULL;
transition->next=NULL;
return transition;
}
Esempio n. 11
0
/**
 * Creates, initializes and returns a transition tagged by an integer.
 */
Transition* new_Transition(int tag_number,int state_number,Transition* next,Abstract_allocator prv_alloc) {
Transition* transition;
transition=(Transition*)malloc_cb(sizeof(Transition),prv_alloc);
if (transition==NULL) {
  fatal_alloc_error("new_Transition");
}
transition->tag_number=tag_number;
transition->state_number=state_number;
transition->next=next;
return transition;
}
Esempio n. 12
0
/**
 * Allocates, initializes and returns a new optimized graph call.
 */
static struct opt_graph_call* new_opt_graph_call(int graph_number,Abstract_allocator prv_alloc) {
struct opt_graph_call* g;
g=(struct opt_graph_call*)malloc_cb(sizeof(struct opt_graph_call),prv_alloc);
if (g==NULL) {
   fatal_alloc_error("new_opt_graph_call");
}
g->graph_number=graph_number;
g->transition=NULL;
g->next=NULL;
return g;
}
Esempio n. 13
0
/**
 * The same than above, except that it does not duplicate the given symbol.
 */
Transition* new_Transition_no_dup(symbol_t* label,int state_number,Transition* next,Abstract_allocator prv_alloc) {
Transition* transition;
transition=(Transition*)malloc_cb(sizeof(Transition),prv_alloc);
if (transition==NULL) {
  fatal_alloc_error("new_Transition");
}
transition->label=label;
transition->state_number=state_number;
transition->next=next;
return transition;
}
Esempio n. 14
0
/**
 * Builds, initializes and returns a new LocateCache.
 */
LocateCache new_LocateCache(int token,struct match_list* matches,Abstract_allocator prv_alloc) {
LocateCache c=(LocateCache)malloc_cb(sizeof(struct locate_cache),prv_alloc);
if (c==NULL) {
	fatal_alloc_error("new_LocateCache");
}
c->left=NULL;
c->middle=NULL;
c->right=NULL;
c->token=token;
c->matches=matches;
return c;
}
Esempio n. 15
0
/**
 * Allocates, initializes and returns a new opt_pattern.
 */
struct opt_pattern* new_opt_pattern(int pattern_number,int negation,Abstract_allocator prv_alloc) {
struct opt_pattern* p;
p=(struct opt_pattern*)malloc_cb(sizeof(struct opt_pattern),prv_alloc);
if (p==NULL) {
   fatal_alloc_error("new_opt_pattern");
}
p->pattern_number=pattern_number;
p->negation=(char)negation;
p->transition=NULL;
p->next=NULL;
return p;
}
/**
 * Allocates, initializes and returns a dictionary node.
 */
struct dictionary_node* new_dictionary_node(Abstract_allocator prv_alloc) {
struct dictionary_node* a=(struct dictionary_node*)malloc_cb(sizeof(struct dictionary_node),prv_alloc);
if (a==NULL) {
	fatal_alloc_error("new_dictionary_node");
}
a->single_INF_code_list=NULL;
a->offset=-1;
a->trans=NULL;
a->incoming=0;
a->n_trans=0;
a->INF_code=0;
return a;
}
Esempio n. 17
0
/**
 * Allocates, initializes and returns a new optimized variable.
 */
static struct opt_variable* new_opt_variable(int variable_number,Transition* transition,
										Abstract_allocator prv_alloc) {
struct opt_variable* v;
v=(struct opt_variable*)malloc_cb(sizeof(struct opt_variable),prv_alloc);
if (v==NULL) {
   fatal_alloc_error("new_opt_variable");
}
v->variable_number=variable_number;
v->transition=NULL;
add_transition_if_not_present(&(v->transition),transition->tag_number,transition->state_number,prv_alloc);
v->next=NULL;
return v;
}
Esempio n. 18
0
/**
 * Allocates, initializes and returns a new optimized meta.
 */
static struct opt_meta* new_opt_meta(enum meta_symbol meta,int negation,Abstract_allocator prv_alloc) {
struct opt_meta* m;
m=(struct opt_meta*)malloc_cb(sizeof(struct opt_meta),prv_alloc);
if (m==NULL) {
   fatal_alloc_error("new_opt_meta");
}
m->meta=meta;
m->negation=(char)negation;
m->transition=NULL;
m->next=NULL;
m->morphological_mode_ends=NULL;
return m;
}
Esempio n. 19
0
/**
 * Allocates, initializes and returns a new pattern.
 */
struct pattern* new_pattern(Abstract_allocator prv_alloc) {
struct pattern* p;
p=(struct pattern*)malloc_cb(sizeof(struct pattern),prv_alloc);
if (p==NULL) {
   fatal_alloc_error("new_pattern");
}
p->inflected=NULL;
p->lemma=NULL;
p->grammatical_codes=NULL;
p->inflectional_codes=NULL;
p->forbidden_codes=NULL;
p->type=UNDEFINED_PATTERN;
return p;
}
Esempio n. 20
0
Transition* clone_transition(const Transition* t,symbol_t*(*clone_elag_symbol)(const symbol_t*),Abstract_allocator prv_alloc) {
Transition* transition;
transition=(Transition*)malloc_cb(sizeof(Transition),prv_alloc);
if (transition==NULL) {
  fatal_alloc_error("clone_transition");
}
if (clone_elag_symbol==NULL) {
   memcpy(transition,t,sizeof(Transition));
} else {
   transition->label=clone_elag_symbol(t->label);
   transition->state_number=t->state_number;
   transition->next=t->next;
}
return transition;
}
Esempio n. 21
0
/**
 * Returns a clone of the pattern.
 */
struct pattern* clone(const struct pattern* src,Abstract_allocator prv_alloc) {
	struct pattern* dst;

	if (src == NULL)
		return NULL;

	dst=(struct pattern*)malloc_cb(sizeof(struct pattern),prv_alloc);
	if (dst==NULL) {
	   fatal_error("Not enough memory in new_pattern_ByCopy\n");
	}
	dst->inflected=u_strdup(src->inflected,prv_alloc);
	dst->lemma=u_strdup(src->lemma,prv_alloc);
	dst->grammatical_codes=clone(src->grammatical_codes,prv_alloc);
	dst->inflectional_codes=clone(src->inflectional_codes,prv_alloc);
	dst->forbidden_codes=clone(src->forbidden_codes,prv_alloc);
	dst->type=src->type;
	return dst;
}
Esempio n. 22
0
/**
 * Allocates, initializes and returns an integer array that contains
 * the elements of the given list. '*size' is set to the size of this
 * array. Note that passing an empty list will return NULL.
 */
int* dump(struct list_int* list,int *size,Abstract_allocator prv_alloc) {
*size=0;
if (list==NULL) return NULL;
struct list_int* tmp=list;
/* We count the number of elements */
while (tmp!=NULL) {
   (*size)++;
   tmp=tmp->next;
}
int* result=(int*)malloc_cb((*size)*sizeof(int),prv_alloc);
if (result==NULL) {
   fatal_alloc_error("dump");
}
tmp=list;
for (int i=0;i<(*size);i++) {
   result[i]=tmp->n;
   tmp=tmp->next;
}
return result;
}
Esempio n. 23
0
/**
 * This function takes a fst2 and returns an array containing the corresponding
 * optimized states.
 */
OptimizedFst2State* build_optimized_fst2_states(Variables* v,OutputVariables* output,Fst2* fst2,
		Abstract_allocator prv_alloc) {
OptimizedFst2State* optimized_states=(OptimizedFst2State*)malloc_cb(fst2->number_of_states*sizeof(OptimizedFst2State),prv_alloc);
if (optimized_states==NULL) {
   fatal_alloc_error("build_optimized_fst2_states");
}

int num_current_graph=1;
int pos_in_current_graph=0;
for (int i=0;i<fst2->number_of_states;i++) {
   optimized_states[i]=optimize_state(v,output,fst2,fst2->states[i],fst2->tags,prv_alloc);

   optimized_states[i]->graph_number=num_current_graph;
   optimized_states[i]->pos_transition_in_fst2=i;
   optimized_states[i]->pos_transition_in_graph=pos_in_current_graph++;

   if (pos_in_current_graph >= *((fst2->number_of_states_per_graphs)+num_current_graph))
   {
	   num_current_graph++;
	   pos_in_current_graph=0;
   }
}

#ifdef AGGRESSIVE_OPTIMIZATION
int n_graphs_emptied;
do {
	n_graphs_emptied=0;
	for (int i=1;i<=fst2->number_of_graphs;i++) {
		n_graphs_emptied+=remove_useless_lexical_transitions(fst2,i,optimized_states,prv_alloc);
	}
} while (n_graphs_emptied!=0);
/* Finally, we convert token lists to sorted array suitable for binary search */
for (int i=0;i<fst2->number_of_states;i++) {
	token_list_2_token_array(optimized_states[i],prv_alloc);
}
#endif // AGGRESSIVE_OPTIMIZATION

return optimized_states;
}
Esempio n. 24
0
/**
 * Allocates, initializes and returns a new match list element.
 */
struct match_list* new_match(int start,int end,int start_char,int end_char,
                             int start_letter,int end_letter,unichar* output,
                             int weight,struct match_list* next,Abstract_allocator prv_alloc) {
struct match_list *l;
l=(struct match_list*)malloc_cb(sizeof(struct match_list),prv_alloc);
if (l==NULL) {
   fatal_alloc_error("new_match");
}
l->m.start_pos_in_token=start;
l->m.end_pos_in_token=end;
l->weight=weight;
if (output==NULL) {
   l->output=NULL;
} else {
   l->output=u_strdup(output,prv_alloc);
}
l->m.start_pos_in_char=start_char;
l->m.end_pos_in_char=end_char;
l->m.start_pos_in_letter=start_letter;
l->m.end_pos_in_letter=end_letter;
l->next=next;
return l;
}
Esempio n. 25
0
connector_handle_t connector_init(connector_callback_t const callback)
{

    connector_data_t * connector_handle = NULL;
    connector_status_t status;

#if (defined CONNECTOR_SW_DESCRIPTION)
    connector_debug_printf("Cloud Connector v%s %s\n", CONNECTOR_SW_VERSION, CONNECTOR_SW_DESCRIPTION);
#else
    connector_debug_printf("Cloud Connector v%s\n", CONNECTOR_SW_VERSION);
#endif

    {
        void * handle;

#if (defined CONNECTOR_NO_MALLOC)
        status = malloc_data_buffer(NULL, sizeof *connector_handle, named_buffer_id(connector_data), &handle);
#else
        status = malloc_cb(callback, sizeof *connector_handle, &handle);
#endif

        COND_ELSE_GOTO(status == connector_working, done);
        memset(handle, 0x00, sizeof *connector_handle); /* Init structure, all pointers to NULL */
        connector_handle = handle;
    }

    connector_handle->callback = callback;

    status = manage_device_id(connector_handle);
    COND_ELSE_GOTO(status == connector_working, error);

    /* make a copy of the cloud url */
#if (defined CONNECTOR_TRANSPORT_TCP) || (defined CONNECTOR_TRANSPORT_UDP)
#if (defined CONNECTOR_CLOUD_URL)
    {
        static char const connector_device_cloud_url[]= CONNECTOR_CLOUD_URL;
        connector_handle->device_cloud_url = (char *)connector_device_cloud_url;
        connector_handle->device_cloud_url_length = sizeof connector_device_cloud_url -1;
    }
#else
    status = get_config_device_cloud_url(connector_handle);
    COND_ELSE_GOTO(status == connector_working, error);
#endif
#endif /* (defined CONNECTOR_TRANSPORT_TCP) || (defined CONNECTOR_TRANSPORT_UDP) */

    /* make a copy of the cloud phone */
#if (defined CONNECTOR_TRANSPORT_SMS)
#if (defined CONNECTOR_CLOUD_PHONE)
    {
        static char const connector_device_cloud_phone[]= CONNECTOR_CLOUD_PHONE;
        connector_handle->device_cloud_phone = (char *)connector_device_cloud_phone;
        connector_handle->device_cloud_phone_length = sizeof connector_device_cloud_phone -1;
    }
#else
    status = get_config_device_cloud_phone(connector_handle);
    COND_ELSE_GOTO(status == connector_working, error);
#endif

#if (defined CONNECTOR_CLOUD_SERVICE_ID)
    {
        static char const connector_device_cloud_service_id[]= CONNECTOR_CLOUD_SERVICE_ID;
        connector_handle->device_cloud_service_id = (char *)connector_device_cloud_service_id;
        connector_handle->device_cloud_service_id_length = sizeof connector_device_cloud_service_id -1;
    }
#else
    status = get_config_device_cloud_service_id(connector_handle);
    COND_ELSE_GOTO(status == connector_working, error);
#endif
#endif /* (defined CONNECTOR_TRANSPORT_SMS) */

#if (defined CONNECTOR_TRANSPORT_TCP)
    status = connector_edp_init(connector_handle);
    COND_ELSE_GOTO(status == connector_working, error);
#endif

#if (defined CONNECTOR_TRANSPORT_UDP) || (defined CONNECTOR_TRANSPORT_SMS)
    status = connector_sm_init(connector_handle);
    COND_ELSE_GOTO(status == connector_working, error);
#endif

#if (defined CONNECTOR_TRANSPORT_COUNT > 1)
    connector_handle->first_running_network = (connector_network_type_t) 0;
#endif

    connector_handle->signature = connector_signature;
    goto done;

error:
    free_data_buffer(connector_handle, named_buffer_id(connector_data), connector_handle);
    connector_handle = NULL;

done:
    return connector_handle;
}
Esempio n. 26
0
int locate_pattern(const char* text_cod,const char* tokens,const char* fst2_name,const char* dlf,const char* dlc,const char* err,
                   const char* alphabet,MatchPolicy match_policy,OutputPolicy output_policy,
                   Encoding encoding_output,int bom_output,int mask_encoding_compatibility_input,
                   const char* dynamicDir,TokenizationPolicy tokenization_policy,
                   SpacePolicy space_policy,int search_limit,const char* morpho_dic_list,
                   AmbiguousOutputPolicy ambiguous_output_policy,
                   VariableErrorPolicy variable_error_policy,int protect_dic_chars,
                   int is_korean,int max_count_call,int max_count_call_warning,
                   char* arabic_rules,int tilde_negation_operator,int useLocateCache,int allow_trace) {

    U_FILE* out;
    U_FILE* info;
    struct locate_parameters* p=new_locate_parameters();
    p->text_cod=af_open_mapfile(text_cod,MAPFILE_OPTION_READ,0);
    p->buffer=(int*)af_get_mapfile_pointer(p->text_cod);
    long text_size=(long)af_get_mapfile_size(p->text_cod)/sizeof(int);
    p->buffer_size=(int)text_size;
    p->tilde_negation_operator=tilde_negation_operator;
    p->useLocateCache=useLocateCache;
    if (max_count_call == -1) {
        max_count_call = (int)text_size;
    }
    if (max_count_call_warning == -1) {
        max_count_call_warning = (int)text_size;
    }
    p->match_policy=match_policy;
    p->tokenization_policy=tokenization_policy;
    p->space_policy=space_policy;
    p->output_policy=output_policy;
    p->search_limit=search_limit;
    p->ambiguous_output_policy=ambiguous_output_policy;
    p->variable_error_policy=variable_error_policy;
    p->protect_dic_chars=protect_dic_chars;
    p->mask_encoding_compatibility_input = mask_encoding_compatibility_input;
    p->max_count_call = max_count_call;
    p->max_count_call_warning = max_count_call_warning;
    p->token_filename = tokens;
    char concord[FILENAME_MAX];
    char concord_info[FILENAME_MAX];

    strcpy(concord,dynamicDir);
    strcat(concord,"concord.ind");

    strcpy(concord_info,dynamicDir);
    strcat(concord_info,"concord.n");

    char morpho_bin[FILENAME_MAX];
    strcpy(morpho_bin,dynamicDir);
    strcat(morpho_bin,"morpho.bin");
    if (arabic_rules!=NULL && arabic_rules[0]!='\0') {
        load_arabic_typo_rules(arabic_rules,&(p->arabic));
    }
    out=u_fopen_versatile_encoding(encoding_output,bom_output,mask_encoding_compatibility_input,concord,U_WRITE);
    if (out==NULL) {
        error("Cannot write %s\n",concord);
        af_release_mapfile_pointer(p->text_cod,p->buffer);
        af_close_mapfile(p->text_cod);
        free_stack_unichar(p->stack);
        free_locate_parameters(p);
        u_fclose(out);
        return 0;
    }
    info=u_fopen_versatile_encoding(encoding_output,bom_output,mask_encoding_compatibility_input,concord_info,U_WRITE);
    if (info==NULL) {
        error("Cannot write %s\n",concord_info);
    }
    switch(output_policy) {
    case IGNORE_OUTPUTS:
        u_fprintf(out,"#I\n");
        break;
    case MERGE_OUTPUTS:
        u_fprintf(out,"#M\n");
        break;
    case REPLACE_OUTPUTS:
        u_fprintf(out,"#R\n");
        break;
    }
    if (alphabet!=NULL && alphabet[0]!='\0') {
        u_printf("Loading alphabet...\n");
        p->alphabet=load_alphabet(alphabet,is_korean);
        if (p->alphabet==NULL) {
            error("Cannot load alphabet file %s\n",alphabet);
            af_release_mapfile_pointer(p->text_cod,p->buffer);
            af_close_mapfile(p->text_cod);
            free_stack_unichar(p->stack);
            free_locate_parameters(p);
            if (info!=NULL) u_fclose(info);
            u_fclose(out);
            return 0;
        }
    }
    struct string_hash* semantic_codes=new_string_hash();
    extract_semantic_codes(dlf,semantic_codes);
    extract_semantic_codes(dlc,semantic_codes);

    if (is_cancelling_requested() != 0) {
        error("user cancel request.\n");
        free_alphabet(p->alphabet);
        free_string_hash(semantic_codes);
        af_release_mapfile_pointer(p->text_cod,p->buffer);
        af_close_mapfile(p->text_cod);
        free_stack_unichar(p->stack);
        free_locate_parameters(p);
        if (info!=NULL) u_fclose(info);
        u_fclose(out);
        return 0;
    }

    u_printf("Loading fst2...\n");
    struct FST2_free_info fst2load_free;
    Fst2* fst2load=load_abstract_fst2(fst2_name,1,&fst2load_free);
    if (fst2load==NULL) {
        error("Cannot load grammar %s\n",fst2_name);
        free_alphabet(p->alphabet);
        free_string_hash(semantic_codes);
        af_release_mapfile_pointer(p->text_cod,p->buffer);
        af_close_mapfile(p->text_cod);
        free_stack_unichar(p->stack);
        free_locate_parameters(p);
        if (info!=NULL) u_fclose(info);
        u_fclose(out);
        return 0;
    }

    Abstract_allocator locate_abstract_allocator=create_abstract_allocator("locate_pattern",AllocatorCreationFlagAutoFreePrefered);


    p->fst2=new_Fst2_clone(fst2load,locate_abstract_allocator);
    free_abstract_Fst2(fst2load,&fst2load_free);

    if (is_cancelling_requested() != 0) {
        error("User cancel request..\n");
        free_alphabet(p->alphabet);
        free_string_hash(semantic_codes);
        free_Fst2(p->fst2,locate_abstract_allocator);
        close_abstract_allocator(locate_abstract_allocator);
        af_release_mapfile_pointer(p->text_cod,p->buffer);
        af_close_mapfile(p->text_cod);
        free_stack_unichar(p->stack);
        free_locate_parameters(p);
        if (info!=NULL) u_fclose(info);
        u_fclose(out);
        return 0;
    }

    p->tags=p->fst2->tags;
#ifdef TRE_WCHAR
    p->filters=new_FilterSet(p->fst2,p->alphabet);
    if (p->filters==NULL) {
        error("Cannot compile filter(s)\n");
        free_alphabet(p->alphabet);
        free_string_hash(semantic_codes);
        free_Fst2(p->fst2,locate_abstract_allocator);
        close_abstract_allocator(locate_abstract_allocator);
        free_stack_unichar(p->stack);
        free_locate_parameters(p);
        af_release_mapfile_pointer(p->text_cod,p->buffer);
        af_close_mapfile(p->text_cod);
        if (info!=NULL) u_fclose(info);
        u_fclose(out);
        return 0;
    }
#endif
    u_printf("Loading token list...\n");
    int n_text_tokens=0;

    p->tokens=load_text_tokens_hash(tokens,mask_encoding_compatibility_input,&(p->SENTENCE),&(p->STOP),&n_text_tokens);
    if (p->tokens==NULL) {
        error("Cannot load token list %s\n",tokens);
        free_alphabet(p->alphabet);
        free_string_hash(semantic_codes);
        free_Fst2(p->fst2,locate_abstract_allocator);
        close_abstract_allocator(locate_abstract_allocator);
        free_locate_parameters(p);
        af_release_mapfile_pointer(p->text_cod,p->buffer);
        af_close_mapfile(p->text_cod);
        if (info!=NULL) u_fclose(info);
        u_fclose(out);
        return 0;
    }
    Abstract_allocator locate_work_abstract_allocator = locate_abstract_allocator;

    p->match_cache=(LocateCache*)malloc_cb(p->tokens->size * sizeof(LocateCache),locate_work_abstract_allocator);
    memset(p->match_cache,0,p->tokens->size * sizeof(LocateCache));
    if (p->match_cache==NULL) {
        fatal_alloc_error("locate_pattern");
    }

#ifdef TRE_WCHAR
    p->filter_match_index=new_FilterMatchIndex(p->filters,p->tokens);
    if (p->filter_match_index==NULL) {
        error("Cannot optimize filter(s)\n");
        free_alphabet(p->alphabet);
        free_string_hash(semantic_codes);
        free_string_hash(p->tokens);
        close_abstract_allocator(locate_abstract_allocator);
        free_locate_parameters(p);
        af_release_mapfile_pointer(p->text_cod,p->buffer);
        af_close_mapfile(p->text_cod);
        if (info!=NULL) u_fclose(info);
        u_fclose(out);
        return 0;
    }
#endif

    if (allow_trace!=0) {
        open_locate_trace(p,&p->fnc_locate_trace_step,&p->private_param_locate_trace);
    }
    extract_semantic_codes_from_tokens(p->tokens,semantic_codes,locate_abstract_allocator);
    u_printf("Loading morphological dictionaries...\n");
    load_morphological_dictionaries(morpho_dic_list,p,morpho_bin);
    extract_semantic_codes_from_morpho_dics(p->morpho_dic_inf,p->n_morpho_dics,semantic_codes,locate_abstract_allocator);
    p->token_control=(unsigned char*)malloc(n_text_tokens*sizeof(unsigned char));
    if (p->token_control==NULL) {
        fatal_alloc_error("locate_pattern");
    }
    p->matching_patterns=(struct bit_array**)malloc(n_text_tokens*sizeof(struct bit_array*));
    if (p->matching_patterns==NULL) {
        fatal_alloc_error("locate_pattern");
    }
    for (int i=0; i<n_text_tokens; i++) {
        p->token_control[i]=0;
        p->matching_patterns[i]=NULL;
    }
    compute_token_controls(p->alphabet,err,p);
    int number_of_patterns,is_DIC,is_CDIC,is_SDIC;
    p->pattern_tree_root=new_pattern_node(locate_abstract_allocator);
    u_printf("Computing fst2 tags...\n");
    process_tags(&number_of_patterns,semantic_codes,&is_DIC,&is_CDIC,&is_SDIC,p,locate_abstract_allocator);
    p->current_compound_pattern=number_of_patterns;
    p->DLC_tree=new_DLC_tree(p->tokens->size);
    struct lemma_node* root=new_lemma_node();
    u_printf("Loading dlf...\n");
    load_dic_for_locate(dlf,mask_encoding_compatibility_input,p->alphabet,number_of_patterns,is_DIC,is_CDIC,root,p);
    u_printf("Loading dlc...\n");
    load_dic_for_locate(dlc,mask_encoding_compatibility_input,p->alphabet,number_of_patterns,is_DIC,is_CDIC,root,p);
    /* We look if tag tokens like "{today,.ADV}" verify some patterns */
    check_patterns_for_tag_tokens(p->alphabet,number_of_patterns,root,p,locate_abstract_allocator);
    u_printf("Optimizing fst2 pattern tags...\n");
    optimize_pattern_tags(p->alphabet,root,p,locate_abstract_allocator);
    u_printf("Optimizing compound word dictionary...\n");
    optimize_DLC(p->DLC_tree);
    free_string_hash(semantic_codes);
    int nb_input_variable=0;
    p->input_variables=new_Variables(p->fst2->input_variables,&nb_input_variable);
    p->output_variables=new_OutputVariables(p->fst2->output_variables,&p->nb_output_variables);


    Abstract_allocator locate_recycle_abstract_allocator=NULL;
    locate_recycle_abstract_allocator=create_abstract_allocator("locate_pattern_recycle",
                                      AllocatorFreeOnlyAtAllocatorDelete|AllocatorTipOftenRecycledObject,
                                      get_prefered_allocator_item_size_for_nb_variable(nb_input_variable));

    u_printf("Optimizing fst2...\n");
    p->optimized_states=build_optimized_fst2_states(p->input_variables,p->output_variables,p->fst2,locate_abstract_allocator);
    if (is_korean) {
        p->korean=new Korean(p->alphabet);
        p->jamo_tags=create_jamo_tags(p->korean,p->tokens);
    }
    p->failfast=new_bit_array(n_text_tokens,ONE_BIT);

    u_printf("Working...\n");
    p->prv_alloc=locate_work_abstract_allocator;
    p->prv_alloc_recycle=locate_recycle_abstract_allocator;
    launch_locate(out,text_size,info,p);
    if (allow_trace!=0) {
        close_locate_trace(p,p->fnc_locate_trace_step,p->private_param_locate_trace);
    }
    free_bit_array(p->failfast);
    free_Variables(p->input_variables);
    free_OutputVariables(p->output_variables);
    af_release_mapfile_pointer(p->text_cod,p->buffer);
    af_close_mapfile(p->text_cod);
    if (info!=NULL) u_fclose(info);
    u_fclose(out);

    if (p->match_cache!=NULL) {
        for (int i=0; i<p->tokens->size; i++) {
            free_LocateCache(p->match_cache[i],locate_work_abstract_allocator);
        }
        free_cb(p->match_cache,locate_work_abstract_allocator);
    }
    int free_abstract_allocator_item=(get_allocator_cb_flag(locate_abstract_allocator) & AllocatorGetFlagAutoFreePresent) ? 0 : 1;

    if (free_abstract_allocator_item) {
        free_optimized_states(p->optimized_states,p->fst2->number_of_states,locate_abstract_allocator);
    }
    free_stack_unichar(p->stack);
    /** Too long to free the DLC tree if it is big
     * free_DLC_tree(p->DLC_tree);
     */
    if (free_abstract_allocator_item) {
        free_pattern_node(p->pattern_tree_root,locate_abstract_allocator);
        free_Fst2(p->fst2,locate_abstract_allocator);
        free_list_int(p->tag_token_list,locate_abstract_allocator);
    }
    close_abstract_allocator(locate_abstract_allocator);
    close_abstract_allocator(locate_recycle_abstract_allocator);
    locate_recycle_abstract_allocator=locate_abstract_allocator=NULL;

    /* We don't free 'parameters->tags' because it was just a link on 'parameters->fst2->tags' */
    free_alphabet(p->alphabet);
    if (p->korean!=NULL) {
        delete p->korean;
    }
    if (p->jamo_tags!=NULL) {
        /* jamo tags must be freed before tokens, because we need to know how
         * many jamo tags there are, and this number is the number of tokens */
        for (int i=0; i<p->tokens->size; i++) {
            free(p->jamo_tags[i]);
        }
        free(p->jamo_tags);
    }
    free_string_hash(p->tokens);
    free_lemma_node(root);
    free(p->token_control);
    for (int i=0; i<n_text_tokens; i++) {
        free_bit_array(p->matching_patterns[i]);
    }
    free(p->matching_patterns);
#ifdef TRE_WCHAR
    free_FilterSet(p->filters);
    free_FilterMatchIndex(p->filter_match_index);
#endif
    for (int i=0; i<p->n_morpho_dics; i++) {
        free_abstract_INF(p->morpho_dic_inf[i],&(p->morpho_dic_inf_free[i]));
        free_abstract_BIN(p->morpho_dic_bin[i],&(p->morpho_dic_bin_free[i]));
    }
    free(p->morpho_dic_inf);
    free(p->morpho_dic_inf_free);
    free(p->morpho_dic_bin);
    free(p->morpho_dic_bin_free);
#if (defined(UNITEX_LIBRARY) || defined(UNITEX_RELEASE_MEMORY_AT_EXIT))
    free_DLC_tree(p->DLC_tree);
#endif
    free_locate_parameters(p);
    u_printf("Done.\n");
    return 1;
}