/* open a file for write */ FILE *gapp_openw(GraceApp *gapp, const char *fn) { struct stat statb; char buf[GR_MAXPATHLEN + 50]; FILE *retval; if (!fn || !fn[0]) { errmsg("No file name given"); return NULL; } else if (strcmp(fn, "-") == 0 || strcmp(fn, "stdout") == 0) { return stdout; } else { if (stat(fn, &statb) == 0) { /* check to make sure this is a file and not a dir */ if (S_ISREG(statb.st_mode)) { sprintf(buf, "Overwrite %s?", fn); if (!yesno(buf, NULL, NULL, NULL)) { return NULL; } } else { sprintf(buf, "%s is not a regular file!", fn); errmsg(buf); return NULL; } } retval = filter_write(gapp, fn); if (!retval) { sprintf(buf, "Can't write to file %s, check permissions!", fn); errmsg(buf); } return retval; } }
int callback_write(const void *buf, int nbyte, void *xtData){ callback_data *cdata; cdata = (callback_data*)xtData; if ( cdata->type == HANDLER ) return ap_rwrite(buf,nbyte,cdata->r); if ( cdata->type == FILTER ) return filter_write((char *)buf,nbyte,cdata); else return 0; }
int zRLD(os_ptr op) { stream *s; int code = filter_write(op, &s_RLD_procs, &s); if ( code < 0 ) return code; s_RLD_init(s); return 0; }
/* <target> <table> <dict> ByteTranslateEncode/filter <file> */ static int zBTE(i_ctx_t *i_ctx_p) { os_ptr op = osp; stream_BT_state bts; int code = bt_setup(op, &bts); if (code < 0) return code; return filter_write(op, 0, &s_BTE_template, (stream_state *)&bts, 0); }
/* <target> <dict> BoundedHuffmanEncode/filter <file> */ static int zBHCE(i_ctx_t *i_ctx_p) { os_ptr op = osp; stream_BHCE_state bhcs; int code = bhc_setup(op, (stream_BHC_state *)&bhcs); if (code < 0) return code; return filter_write(op, 0, &s_BHCE_template, (stream_state *)&bhcs, 0); }
/* <target> <seed> <dict_ignored> eexecEncode/filter <file> */ static int zexE(i_ctx_t *i_ctx_p) { os_ptr op = osp; stream_exE_state state; int code = eexec_param(op, &state.cstate); if (code < 0) return code; return filter_write(i_ctx_p, code, &s_exE_template, (stream_state *)&state, 0); }
int zRLE(register os_ptr op) { stream *s; int code; check_type(*op, t_integer); if ( (ulong)(op->value.intval) > max_uint ) return e_rangecheck; code = filter_write(op - 1, &s_RLE_procs, &s); if ( code < 0 ) return code; s_RLE_init(s, (uint)(op->value.intval)); pop(1); return 0; }
/* <target> <dict> BWBlockSortEncode/filter <file> */ static int zBWBSE(i_ctx_t *i_ctx_p) { os_ptr op = osp; stream_BWBSE_state bwbss; int code; check_type(*op, t_dictionary); check_dict_read(*op); code = bwbs_setup(op, (stream_BWBS_state *)&bwbss); if (code < 0) return code; return filter_write(op, 0, &s_BWBSE_template, (stream_state *)&bwbss, 0); }
/* <target> <dict> CCITTFaxEncode/filter <file> */ static int zCFE(i_ctx_t *i_ctx_p) { os_ptr op = osp; stream_CFE_state cfs; int code; check_type(*op, t_dictionary); check_dict_read(*op); code = zcf_setup(op, (stream_CF_state *)&cfs, iimemory); if (code < 0) return code; return filter_write(i_ctx_p, 0, &s_CFE_template, (stream_state *)&cfs, 0); }
/* encode version of the filter */ static int z_arcfour_e(i_ctx_t * i_ctx_p) { os_ptr op = osp; /* i_ctx_p->op_stack.stack.p defined in osstack.h */ ref *sop = NULL; stream_arcfour_state state; /* extract the key from the parameter dictionary */ check_type(*op, t_dictionary); check_dict_read(*op); if (dict_find_string(op, "Key", &sop) <= 0) return_error(e_rangecheck); s_arcfour_set_key(&state, sop->value.const_bytes, r_size(sop)); /* we pass npop=0, since we've no arguments left to consume */ /* we pass 0 instead of the usual rspace(sop) will allocate storage for filter state from the same memory pool as the stream it's coding. this causes no trouble because we maintain no pointers */ return filter_write(i_ctx_p, 0, &s_arcfour_template, (stream_state *) & state, 0); }
ssize_t write(int fd, const void *buf, size_t count) { if (fd != STDOUT_FILENO || !filter_write) return libc_write(fd, buf, count); return filter_write(current_write_filter, buf, count); }
int zNullE(os_ptr op) { return filter_write(op, &s_NullE_procs, NULL); }
int zA85E(os_ptr op) { return filter_write(op, &s_A85E_procs, NULL); }
void M2_p2o2::each_train_one_iter() { static bool** STA_noprobs = 0; //static ine, init only once if(STA_noprobs==0 && !filter_read(STA_noprobs)){ //init only once int all_tokens_train=0,all_token_filter_wrong=0; time_t now; time(&now); cout << "-Preparing no_probs at " << ctime(&now) << endl; STA_noprobs = new bool*[training_corpus->size()]; for(unsigned int i=0;i<training_corpus->size();i++){ DependencyInstance* x = training_corpus->at(i); STA_noprobs[i] = get_cut_o1(x,mfo1,dict,hp->CONF_score_o1filter_cut); all_tokens_train += x->length()-1; for(int m=1;m<x->length();m++) if(STA_noprobs[i][get_index2(x->length(),x->heads->at(m),m)]) all_token_filter_wrong ++; } cout << "For o1 filter: all " << all_tokens_train << ";filter wrong " << all_token_filter_wrong << endl; filter_write(STA_noprobs); } //per-sentence approach int num_sentences = training_corpus->size(); //statistics int skip_sent_num = 0; int all_forward_instance = 0; int all_inst_right = 0; int all_inst_wrong = 0; //some useful info int odim = mach->get_odim(); //training time_t now; time(&now); //ctime is not rentrant ! use ctime_r() instead if needed cout << "##*** // Start the p2o2 training for iter " << cur_iter << " at " << ctime(&now) << "with lrate " << cur_lrate << endl; cout << "#Sentences is " << num_sentences << " and resample (about)" << num_sentences*hp->CONF_NN_resample << endl; for(int i=0;i<num_sentences;){ //random skip (instead of shuffling every time) if(drand48() > hp->CONF_NN_resample || training_corpus->at(i)->length() >= hp->CONF_higho_toolong){ skip_sent_num ++; i ++; continue; } mach->prepare_batch(); //if nesterov update before each batch (pre-update) if(hp->CONF_NESTEROV_MOMENTUM) mach->nesterov_update(hp->CONF_UPDATE_WAY,hp->CONF_MOMENTUM_ALPHA); //main batch int this_sentence = 0; int this_instance = 0; for(;;){ //forward DependencyInstance* x = training_corpus->at(i); const int length = x->length(); nn_input* the_inputs; REAL *fscores = forward_scores_o2sib(x,mach,&the_inputs,dict->get_helper(),0,STA_noprobs[i],hp); this_instance += the_inputs->get_numi(); all_forward_instance += the_inputs->get_numi(); all_inst_right += the_inputs->inst_good; all_inst_wrong += the_inputs->inst_bad; this_sentence ++; i++; the_scores::Scores<REAL_SCORES>* rscores = get_the_scores(the_inputs,fscores,mach->get_odim(),the_inputs->get_numi()); REAL_SCORES* tmp_marginals = LencodeMarginals_o2sib(length,*rscores); // //two situations // int length = x->length(); // if(!hp->CONF_labeled){ // //calculate prob // rscores = rearrange_scores_o2sib(x,mach,the_inputs,fscores,0,0,0,hp); // tmp_marginals = encodeMarginals_o2sib(length,rscores); // } // else{ // //calculate prob // rscores = rearrange_scores_o2sib(x,mach,the_inputs,fscores,0,0,0,hp); // tmp_marginals = LencodeMarginals_o2sib(length,rscores,mach->get_odim()); // } //set gradients int HERE_dim = the_inputs->num_width; REAL* to_assign = fscores; for(int ii=0;ii<the_inputs->num_inst*HERE_dim;ii+=HERE_dim){ int tmph = the_inputs->inputs->at(ii); int tmpm = the_inputs->inputs->at(ii+1); int tmps = the_inputs->inputs->at(ii+2); if(tmps<0) tmps = tmph; int tmp_goal = the_inputs->goals->at(ii/HERE_dim); REAL_SCORES* from_mar = tmp_marginals+odim*(ii/HERE_dim); for(int once=0;once<odim;once++,to_assign++){ if(tmp_goal == once) *to_assign = -1 * (1 - from_mar[once]) + *to_assign * hp->CONF_score_p2reg; else *to_assign = from_mar[once] + *to_assign * hp->CONF_score_p2reg; //now object is maximum } } //backward mach->backward(fscores); delete the_inputs; delete rscores; delete []tmp_marginals; if(i>=num_sentences) break; //out of the mini-batch while(training_corpus->at(i)->length() >= hp->CONF_higho_toolong){ //HAVE to compromise, bad choice skip_sent_num ++; i ++; } if(i>=num_sentences) break; if(hp->CONF_minibatch > 0){ if(this_sentence >= hp->CONF_minibatch) break; } else{ if(this_instance >= -1*hp->CONF_minibatch) break; } } //real update mach->update(hp->CONF_UPDATE_WAY,cur_lrate,hp->CONF_NN_WD,hp->CONF_MOMENTUM_ALPHA,hp->CONF_RMS_SMOOTH); } cout << "Iter done, skip " << skip_sent_num << " sentences and f&b " << all_forward_instance << ";good/bad: " << all_inst_right << "/" << all_inst_wrong << endl; }