void SENNA_fread_tensor_2d(float **ptr, int *n_row, int *n_column, FILE *stream) { SENNA_fread(n_row, sizeof(int), 1, stream); SENNA_fread(n_column, sizeof(int), 1, stream); *ptr = SENNA_malloc(sizeof(float), (*n_row)*(*n_column)); SENNA_fread(*ptr, sizeof(float), (*n_row)*(*n_column), stream); }
SENNA_CHK *SENNA_CHK_new(const char *path, const char *subpath) { SENNA_CHK *chk = SENNA_malloc(sizeof(SENNA_CHK), 1); FILE *f; float dummy; memset(chk, 0, sizeof(SENNA_CHK)); f = SENNA_fopen(path, subpath, "rb"); SENNA_fread(&chk->window_size, sizeof(int), 1, f); SENNA_fread_tensor_2d(&chk->ll_word_weight, &chk->ll_word_size, &chk->ll_word_max_idx, f); SENNA_fread_tensor_2d(&chk->ll_caps_weight, &chk->ll_caps_size, &chk->ll_caps_max_idx, f); SENNA_fread_tensor_2d(&chk->ll_posl_weight, &chk->ll_posl_size, &chk->ll_posl_max_idx, f); SENNA_fread_tensor_2d(&chk->l1_weight, &chk->input_state_size, &chk->hidden_state_size, f); SENNA_fread_tensor_1d(&chk->l1_bias, &chk->hidden_state_size, f); SENNA_fread_tensor_2d(&chk->l2_weight, &chk->hidden_state_size, &chk->output_state_size, f); SENNA_fread_tensor_1d(&chk->l2_bias, &chk->output_state_size, f); SENNA_fread_tensor_1d(&chk->viterbi_score_init, &chk->output_state_size, f); SENNA_fread_tensor_2d(&chk->viterbi_score_trans, &chk->output_state_size, &chk->output_state_size, f); SENNA_fread(&chk->ll_word_padding_idx, sizeof(int), 1, f); SENNA_fread(&chk->ll_caps_padding_idx, sizeof(int), 1, f); SENNA_fread(&chk->ll_posl_padding_idx, sizeof(int), 1, f); SENNA_fread(&dummy, sizeof(float), 1, f); SENNA_fclose(f); if ((int)dummy != 777) SENNA_error("chk: data corrupted (or not IEEE floating computer)"); chk->input_state = NULL; chk->hidden_state = SENNA_malloc(sizeof(float), chk->hidden_state_size); chk->output_state = NULL; chk->labels = NULL; /* some info if you want verbose */ SENNA_message("chk: window size: %d", chk->window_size); SENNA_message("chk: vector size in word lookup table: %d", chk->ll_word_size); SENNA_message("chk: word lookup table size: %d", chk->ll_word_max_idx); SENNA_message("chk: vector size in caps lookup table: %d", chk->ll_caps_size); SENNA_message("chk: caps lookup table size: %d", chk->ll_caps_max_idx); SENNA_message("chk: vector size in pos lookup table: %d", chk->ll_posl_size); SENNA_message("chk: pos lookup table size: %d", chk->ll_posl_max_idx); SENNA_message("chk: number of hidden units: %d", chk->hidden_state_size); SENNA_message("chk: number of classes: %d", chk->output_state_size); return chk; }
SENNA_PSG* SENNA_PSG_new(const char *path, const char *subpath) { SENNA_PSG *psg = SENNA_malloc(sizeof(SENNA_PSG), 1); FILE *f; float dummy; memset(psg, 0, sizeof(SENNA_PSG)); f = SENNA_fopen(path, subpath, "rb"); SENNA_fread_tensor_2d(&psg->ll_word_weight, &psg->ll_word_size, &psg->ll_word_max_idx, f); SENNA_fread_tensor_2d(&psg->ll_caps_weight, &psg->ll_caps_size, &psg->ll_caps_max_idx, f); SENNA_fread_tensor_2d(&psg->ll_posl_weight, &psg->ll_posl_size, &psg->ll_posl_max_idx, f); SENNA_fread_tensor_2d(&psg->ll_psgl_weight, &psg->ll_psgl_size, &psg->ll_psgl_max_idx, f); SENNA_fread_tensor_2d(&psg->l1_weight, &psg->input_state_size, &psg->l1_state_size, f); SENNA_fread_tensor_1d(&psg->l1_bias, &psg->l1_state_size, f); SENNA_fread_tensor_2d(&psg->l2_bias, &psg->l1_state_size, &psg->window_size, f); SENNA_fread_tensor_2d(&psg->l3_weight, &psg->l2_state_size, &psg->l3_state_size, f); SENNA_fread_tensor_1d(&psg->l3_bias, &psg->l3_state_size, f); SENNA_fread_tensor_2d(&psg->l4_weight, &psg->l3_state_size, &psg->l4_state_size, f); SENNA_fread_tensor_1d(&psg->l4_bias, &psg->l4_state_size, f); SENNA_fread_tensor_1d(&psg->viterbi_score_init, &psg->l4_state_size, f); SENNA_fread_tensor_2d(&psg->viterbi_score_trans, &psg->l4_state_size, &psg->l4_state_size, f); SENNA_fread(&psg->ll_word_padding_idx, sizeof(int), 1, f); SENNA_fread(&psg->ll_caps_padding_idx, sizeof(int), 1, f); SENNA_fread(&psg->ll_posl_padding_idx, sizeof(int), 1, f); SENNA_fread(&psg->ll_psgl_padding_idx, sizeof(int), 1, f); SENNA_fread(&dummy, sizeof(float), 1, f); SENNA_fclose(f); if((int)dummy != 777) SENNA_error("psg: data corrupted (or not IEEE floating computer)"); psg->input_state = NULL; psg->l1_state = NULL; psg->l2_state = NULL; psg->l3_state = NULL; psg->l4_state = NULL; psg->labels = NULL; psg->treillis = SENNA_Treillis_new(); return psg; }
SENNA_Hash *SENNA_Hash_new_with_admissible_keys(const char *path, const char *filename, const char *admissible_keys_filename) { SENNA_Hash *hash = SENNA_Hash_new(path, filename); FILE *f; int admissiblekeyssize = 0; f = SENNA_fopen(path, admissible_keys_filename, "rb"); //@Aure b means that the file is binary file. SENNA_fseek(f, 0, SEEK_END); //@Aure #define SEEK_END 2 Reposition stream position indicator admissiblekeyssize = SENNA_ftell(f); //@ Get current position in stream if(admissiblekeyssize != hash->size) SENNA_error("inconsistent hash and admissible key files"); SENNA_fseek(f, 0, SEEK_SET); hash->is_admissible_key = SENNA_malloc(sizeof(char), admissiblekeyssize); SENNA_fread(hash->is_admissible_key, 1, admissiblekeyssize, f); //@ Read block of data from stream to char array is_admissible_key. SENNA_fclose(f); return hash; }
void SENNA_fread_tensor_1d(float **ptr, int *n_row, FILE *stream) { SENNA_fread(n_row, sizeof(int), 1, stream); *ptr = SENNA_malloc(sizeof(float), *n_row); SENNA_fread(*ptr, sizeof(float), *n_row, stream); }
SENNA_SRL *SENNA_SRL_new(const char *path, const char *subpath) { SENNA_SRL *srl = SENNA_malloc(sizeof(SENNA_SRL), 1); FILE *f; float dummy; int dummy_size; f = SENNA_fopen(path, subpath, "rb"); SENNA_fread(&srl->window_size, sizeof(int), 1, f); SENNA_fread_tensor_2d(&srl->ll_word_weight, &srl->ll_word_size, &srl->ll_word_max_idx, f); SENNA_fread_tensor_2d(&srl->ll_caps_weight, &srl->ll_caps_size, &srl->ll_caps_max_idx, f); SENNA_fread_tensor_2d(&srl->ll_chkl_weight, &srl->ll_chkl_size, &srl->ll_chkl_max_idx, f); SENNA_fread_tensor_2d(&srl->ll_posv_weight, &srl->ll_posv_size, &srl->ll_posv_max_idx, f); SENNA_fread_tensor_2d(&srl->ll_posw_weight, &srl->ll_posw_size, &srl->ll_posw_max_idx, f); SENNA_fread_tensor_2d(&srl->l1_weight_wcc, &dummy_size, &srl->hidden_state1_size, f); SENNA_fread_tensor_2d(&srl->l1_weight_pv, &dummy_size, &srl->hidden_state1_size, f); SENNA_fread_tensor_2d(&srl->l1_weight_pw, &dummy_size, &srl->hidden_state1_size, f); SENNA_fread_tensor_1d(&srl->l1_bias, &srl->hidden_state1_size, f); SENNA_fread_tensor_2d(&srl->l3_weight, &srl->hidden_state1_size, &srl->hidden_state3_size, f); SENNA_fread_tensor_1d(&srl->l3_bias, &srl->hidden_state3_size, f); SENNA_fread_tensor_2d(&srl->l4_weight, &srl->hidden_state3_size, &srl->output_state_size, f); SENNA_fread_tensor_1d(&srl->l4_bias, &srl->output_state_size, f); SENNA_fread_tensor_1d(&srl->viterbi_score_init, &srl->output_state_size, f); SENNA_fread_tensor_2d(&srl->viterbi_score_trans, &srl->output_state_size, &srl->output_state_size, f); SENNA_fread(&srl->ll_word_padding_idx, sizeof(int), 1, f); SENNA_fread(&srl->ll_caps_padding_idx, sizeof(int), 1, f); SENNA_fread(&srl->ll_chkl_padding_idx, sizeof(int), 1, f); SENNA_fread(&dummy, sizeof(float), 1, f); SENNA_fclose(f); if ((int)dummy != 777) SENNA_error("srl: data corrupted (or not IEEE floating computer)"); /* states */ srl->sentence_posv = NULL; srl->sentence_posw = NULL; srl->input_state = NULL; srl->input_state_wcc = NULL; srl->input_state_pv = NULL; srl->input_state_pw = NULL; srl->hidden_state1 = NULL; srl->hidden_state1_wcc = NULL; srl->hidden_state1_pv = NULL; srl->hidden_state1_pw = NULL; srl->hidden_state2 = NULL; srl->hidden_state3 = NULL; srl->output_state = NULL; srl->labels = NULL; srl->labels_size = 0; srl->service = false; srl->debug = false; srl->calls = 0; srl->dnntime = 0; srl->apptime = 0; /* some info if you want verbose */ SENNA_message("srl: window size: %d", srl->window_size); SENNA_message("srl: vector size in word lookup table: %d", srl->ll_word_size); SENNA_message("srl: word lookup table size: %d", srl->ll_word_max_idx); SENNA_message("srl: vector size in caps lookup table: %d", srl->ll_caps_size); SENNA_message("srl: caps lookup table size: %d", srl->ll_caps_max_idx); SENNA_message("srl: vector size in verb position lookup table: %d", srl->ll_posv_size); SENNA_message("srl: verb position lookup table size: %d", srl->ll_posv_max_idx); SENNA_message("srl: vector size in word position lookup table: %d", srl->ll_posw_size); SENNA_message("srl: word position lookup table size: %d", srl->ll_posw_max_idx); SENNA_message("srl: number of hidden units (convolution): %d", srl->hidden_state1_size); SENNA_message("srl: number of hidden units (hidden layer): %d", srl->hidden_state3_size); SENNA_message("srl: number of classes: %d", srl->output_state_size); return srl; }