static int
z_aes_d(i_ctx_t * i_ctx_p)
{
os_ptr op = osp; /* i_ctx_p->op_stack.stack.p defined in osstack.h */
ref *sop = NULL;
stream_aes_state state;
int use_padding;
/* extract the key from the parameter dictionary */
check_type(*op, t_dictionary);
check_dict_read(*op);
if (dict_find_string(op, "Key", &sop) <= 0)
return_error(gs_error_rangecheck);
s_aes_set_key(&state, sop->value.const_bytes, r_size(sop));
/* extract the padding flag, which defaults to true for compatibility */
if (dict_bool_param(op, "Padding", 1, &use_padding) < 0)
return_error(gs_error_rangecheck);
s_aes_set_padding(&state, use_padding);
/* we pass npop=0, since we've no arguments left to consume */
/* FIXME: passing 0 instead of the usual rspace(sop) will allocate
storage for filter state from the same memory pool as the stream
it's coding. this caused no trouble when we were the arcfour cipher
and maintained no pointers. */
return filter_read(i_ctx_p, 0, &s_aes_template,
(stream_state *) & state, 0);
}
/* <source> <dict> /JBIG2Decode <file> */
static int
z_jbig2decode(i_ctx_t * i_ctx_p)
{
os_ptr op = osp;
ref *sop = NULL;
s_jbig2_global_data_t *gref;
stream_jbig2decode_state state;
/* Extract the global context reference, if any, from the parameter
dictionary and embed it in our stream state. The original object
ref is under the JBIG2Globals key.
We expect the postscript code to resolve this and call
z_jbig2makeglobalctx() below to create an astruct wrapping the
global decoder data and store it under the .jbig2globalctx key
*/
s_jbig2decode_set_global_data((stream_state*)&state, NULL);
if (r_has_type(op, t_dictionary)) {
check_dict_read(*op);
if ( dict_find_string(op, ".jbig2globalctx", &sop) > 0) {
gref = r_ptr(sop, s_jbig2_global_data_t);
s_jbig2decode_set_global_data((stream_state*)&state, gref);
}
}
/* we pass npop=0, since we've no arguments left to consume */
return filter_read(i_ctx_p, 0, &s_jbig2decode_template,
(stream_state *) & state, (sop ? r_space(sop) : 0));
}
static int
z_imscale_d(i_ctx_t * i_ctx_p)
{
os_ptr op = osp; /* i_ctx_p->op_stack.stack.p defined in osstack.h */
int width, height;
stream_imscale_state state;
/* extract the key from the parameter dictionary */
check_type(*op, t_dictionary);
check_dict_read(*op);
if (dict_int_param(op, "Width", 0, 1<<24, -1, &width) < 0)
return_error(gs_error_rangecheck);
if (dict_int_param(op, "Height", 0, 1<<24, -1, &height) < 0)
return_error(gs_error_rangecheck);
state.params.spp_decode = 1;
state.params.spp_interp = 1;
state.params.BitsPerComponentIn = 1;
state.params.MaxValueIn = 1;
state.params.WidthIn = width;
state.params.HeightIn = height;
state.params.BitsPerComponentOut = 1;
state.params.MaxValueOut = 1;
state.params.WidthOut = width << 2;
state.params.HeightOut = height << 2;
/* we pass npop=0, since we've no arguments left to consume */
/* we pass 0 instead of the usual rspace(sop) will allocate storage for
filter state from the same memory pool as the stream it's coding. this
causes no trouble because we maintain no pointers */
return filter_read(i_ctx_p, 0, &s_imscale_template,
(stream_state *) & state, 0);
}
/* <target> <dict> TBCPDecode/filter <file> */
static int
zTBCPD(i_ctx_t *i_ctx_p)
{
stream_BCPD_state state;
state.signal_interrupt = no_bcp_signal_interrupt;
state.request_status = no_bcp_request_status;
return filter_read(i_ctx_p, 0, &s_TBCPD_template, (stream_state *)&state, 0);
}
/* <source> <dict> BWBlockSortDecode/filter <file> */
static int
zBWBSD(i_ctx_t *i_ctx_p)
{
os_ptr op = osp;
stream_BWBSD_state bwbss;
int code = bwbs_setup(op, (stream_BWBS_state *)&bwbss);
if (code < 0)
return code;
return filter_read(i_ctx_p, 0, &s_BWBSD_template, (stream_state *)&bwbss, 0);
}
/* <source> <dict> BoundedHuffmanDecode/filter <file> */
static int
zBHCD(i_ctx_t *i_ctx_p)
{
os_ptr op = osp;
stream_BHCD_state bhcs;
int code = bhc_setup(op, (stream_BHC_state *)&bhcs);
if (code < 0)
return code;
return filter_read(i_ctx_p, 0, &s_BHCD_template, (stream_state *)&bhcs, 0);
}
/* <source> DCTDecode/filter <file> */
static int
zDCTD(i_ctx_t *i_ctx_p)
{
os_ptr op = osp;
gs_memory_t *mem;
stream_DCT_state state;
dict_param_list list;
jpeg_decompress_data *jddp;
int code;
const ref *dop;
uint dspace;
if (r_has_type(op, t_dictionary))
dop = op, dspace = r_space(op);
else
dop = 0, dspace = 0;
mem = (gs_memory_t *)find_stream_memory(i_ctx_p, 0, &dspace);
state.memory = mem;
/* First allocate space for IJG parameters. */
jddp = gs_alloc_struct_immovable(mem,jpeg_decompress_data,
&st_jpeg_decompress_data, "zDCTD");
if (jddp == 0)
return_error(e_VMerror);
if (s_DCTD_template.set_defaults)
(*s_DCTD_template.set_defaults) ((stream_state *) & state);
state.data.decompress = jddp;
jddp->memory = state.jpeg_memory = mem; /* set now for allocation */
jddp->scanline_buffer = NULL; /* set this early for safe error exit */
state.report_error = filter_report_error; /* in case create fails */
if ((code = gs_jpeg_create_decompress(&state)) < 0)
goto fail; /* correct to do jpeg_destroy here */
/* Read parameters from dictionary */
if ((code = dict_param_list_read(&list, dop, NULL, false, iimemory)) < 0)
goto fail;
if ((code = s_DCTD_put_params((gs_param_list *) & list, &state)) < 0)
goto rel;
/* Create the filter. */
jddp->templat = s_DCTD_template;
code = filter_read(i_ctx_p, 0, &jddp->templat,
(stream_state *) & state, dspace);
if (code >= 0) /* Success! */
return code;
/*
* We assume that if filter_read fails, the stream has not been
* registered for closing, so s_DCTD_release will never be called.
* Therefore we free the allocated memory before failing.
*/
rel:
iparam_list_release(&list);
fail:
gs_jpeg_destroy(&state);
gs_free_object(mem, jddp, "zDCTD fail");
return code;
}
/* <target> <table> <dict> ByteTranslateDecode/filter <file> */
static int
zBTD(i_ctx_t *i_ctx_p)
{
os_ptr op = osp;
stream_BT_state bts;
int code = bt_setup(op, &bts);
if (code < 0)
return code;
return filter_read(i_ctx_p, 0, &s_BTD_template, (stream_state *)&bts, 0);
}
int
zSFD(os_ptr op)
{ int code;
check_type(op[-1], t_integer);
check_type(*op, t_string);
if ( op[-1].value.intval < 0 ) return e_rangecheck;
if ( op[-1].value.intval != 0 || r_size(op) != 0 )
return e_undefined; /* NYI */
code = filter_read(op - 2, &s_SFD_procs, NULL);
if ( code < 0 ) return code;
pop(2);
return 0;
}
/* open a file for read */
FILE *gapp_openr(GraceApp *gapp, const char *fn, int src)
{
struct stat statb;
char *tfn;
char buf[GR_MAXPATHLEN + 50];
FILE *fp;
if (!fn || !fn[0]) {
errmsg("No file name given");
return NULL;
}
switch (src) {
case SOURCE_DISK:
tfn = grace_path(gapp->grace, fn);
if (strcmp(tfn, "-") == 0 || strcmp(tfn, "stdin") == 0) {
xfree(tfn);
return stdin;
} else if (stat(tfn, &statb)) {
sprintf(buf, "Can't stat file %s", tfn);
errmsg(buf);
xfree(tfn);
return NULL;
/* check to make sure this is a file and not a dir */
} else if (!S_ISREG(statb.st_mode)) {
sprintf(buf, "%s is not a regular file", tfn);
errmsg(buf);
xfree(tfn);
return NULL;
} else {
fp = filter_read(gapp, tfn);
xfree(tfn);
return fp;
}
break;
case SOURCE_PIPE:
tfn = gapp_exe_path(gapp, fn);
fp = popen(tfn, "r");
xfree(tfn);
return fp;
break;
default:
errmsg("Wrong call to gapp_openr()");
return NULL;
}
}
/* <source> <dict> /JPXDecode <file> */
private int
z_jpx_decode(i_ctx_t * i_ctx_p)
{
os_ptr op = osp;
ref *sop = NULL;
stream_jpxd_state state;
state.jpx_memory = imemory->non_gc_memory;
if (r_has_type(op, t_dictionary)) {
check_dict_read(*op);
if ( dict_find_string(op, "Colorspace", &sop) > 0) {
dlprintf("found Colorspace parameter (NYI)\n");
}
}
/* we pass npop=0, since we've no arguments left to consume */
/* we pass 0 instead of the usual rspace(sop) which will allocate storage
for filter state from the same memory pool as the stream it's coding.
this causes no trouble because we maintain no pointers */
return filter_read(i_ctx_p, 0, &s_jpxd_template,
(stream_state *) & state, 0);
}
static int
z_arcfour_d(i_ctx_t * i_ctx_p)
{
os_ptr op = osp; /* i_ctx_p->op_stack.stack.p defined in osstack.h */
ref *sop = NULL;
stream_arcfour_state state;
/* extract the key from the parameter dictionary */
check_type(*op, t_dictionary);
check_dict_read(*op);
if (dict_find_string(op, "Key", &sop) <= 0)
return_error(e_rangecheck);
s_arcfour_set_key(&state, sop->value.const_bytes, r_size(sop));
/* we pass npop=0, since we've no arguments left to consume */
/* we pass 0 instead of the usual rspace(sop) will allocate storage for
filter state from the same memory pool as the stream it's coding. this
causes no trouble because we maintain no pointers */
return filter_read(i_ctx_p, 0, &s_arcfour_template,
(stream_state *) & state, 0);
}
/*
* Initializes a linked list of strings containing hosts/urls to be filtered
*/
void filter_init(void)
{
struct filter_list *p = NULL, *q;
struct filter_s *f;
if (!config.filters) {
filterlist_initialized = 1;
return;
}
if (!fl && !filterlist_initialized) {
int i = 0;
while ((f = config.filters[i++])) {
/* are there already rules for this acl? */
if (!(q = filter_get(f->aclname))) {
log_message(LOG_INFO, "%s: New Filter for %s", __func__, f->aclname);
if (!p) { /* head of list */
p = fl = safecalloc(1, sizeof(struct filter_list));
} else { /* next entry */
p->next = safecalloc(1, sizeof(struct filter_list));
p = p->next;
}
p->aclname = safestrdup(f->aclname);
q = p;
} else {
log_message(LOG_INFO, "%s: Found Filter for %s", __func__, f->aclname);
}
filter_read(f->expression, &q->rules);
}
filterlist_initialized = 1;
}
}
/* <source> <dict> /JPXDecode <file> */
static int
z_jpx_decode(i_ctx_t * i_ctx_p)
{
os_ptr op = osp;
ref *sop = NULL;
ref *csname = NULL;
stream_jpxd_state state;
/* it's our responsibility to call set_defaults() */
state.memory = imemory->non_gc_memory;
if (s_jpxd_template.set_defaults)
(*s_jpxd_template.set_defaults)((stream_state *)&state);
if (r_has_type(op, t_dictionary)) {
check_dict_read(*op);
if ( dict_find_string(op, "Alpha", &sop) > 0) {
check_type(*sop, t_boolean);
if (sop->value.boolval)
state.alpha = true;
}
if ( dict_find_string(op, "ColorSpace", &sop) > 0) {
/* parse the value */
if (r_is_array(sop)) {
/* assume it's the first array element */
csname = sop->value.refs;
} else if (r_has_type(sop,t_name)) {
/* use the name directly */
csname = sop;
} else {
dmprintf(imemory, "warning: JPX ColorSpace value is an unhandled type!\n");
}
if (csname != NULL) {
ref sref;
/* get a reference to the name's string value */
name_string_ref(imemory, csname, &sref);
/* request raw index values if the colorspace is /Indexed */
if (!ISTRCMP(&sref, "Indexed"))
state.colorspace = gs_jpx_cs_indexed;
/* tell the filter what output we want for other spaces */
else if (!ISTRCMP(&sref, "DeviceGray"))
state.colorspace = gs_jpx_cs_gray;
else if (!ISTRCMP(&sref, "DeviceRGB"))
state.colorspace = gs_jpx_cs_rgb;
else if (!ISTRCMP(&sref, "DeviceCMYK"))
state.colorspace = gs_jpx_cs_cmyk;
else if (!ISTRCMP(&sref, "ICCBased")) {
/* The second array element should be the profile's
stream dict */
ref *csdict = sop->value.refs + 1;
ref *nref;
ref altname;
if (r_is_array(sop) && (r_size(sop) > 1) &&
r_has_type(csdict, t_dictionary)) {
check_dict_read(*csdict);
/* try to look up the alternate space */
if (dict_find_string(csdict, "Alternate", &nref) > 0) {
name_string_ref(imemory, csname, &altname);
if (!ISTRCMP(&altname, "DeviceGray"))
state.colorspace = gs_jpx_cs_gray;
else if (!ISTRCMP(&altname, "DeviceRGB"))
state.colorspace = gs_jpx_cs_rgb;
else if (!ISTRCMP(&altname, "DeviceCMYK"))
state.colorspace = gs_jpx_cs_cmyk;
}
/* else guess based on the number of components */
if (state.colorspace == gs_jpx_cs_unset &&
dict_find_string(csdict, "N", &nref) > 0) {
if_debug1m('w', imemory, "[w] JPX image has an external %"PRIpsint
" channel colorspace\n", nref->value.intval);
switch (nref->value.intval) {
case 1: state.colorspace = gs_jpx_cs_gray;
break;
case 3: state.colorspace = gs_jpx_cs_rgb;
break;
case 4: state.colorspace = gs_jpx_cs_cmyk;
break;
}
}
}
}
} else {
if_debug0m('w', imemory, "[w] Couldn't read JPX ColorSpace key!\n");
}
}
}
/* we pass npop=0, since we've no arguments left to consume */
/* we pass 0 instead of the usual rspace(sop) which will allocate storage
for filter state from the same memory pool as the stream it's coding.
this causes no trouble because we maintain no pointers */
return filter_read(i_ctx_p, 0, &s_jpxd_template,
(stream_state *) & state, 0);
}
/* <source> <dict> eexecDecode/filter <file> */
static int
zexD(i_ctx_t *i_ctx_p)
{
os_ptr op = osp;
stream_exD_state state;
int code;
(*s_exD_template.set_defaults)((stream_state *)&state);
if (r_has_type(op, t_dictionary)) {
uint cstate;
bool is_eexec;
check_dict_read(*op);
if ((code = dict_uint_param(op, "seed", 0, 0xffff, 0x10000,
&cstate)) < 0 ||
(code = dict_int_param(op, "lenIV", 0, max_int, 4,
&state.lenIV)) < 0 ||
(code = dict_bool_param(op, "eexec", false,
&is_eexec)) < 0 ||
(code = dict_bool_param(op, "keep_spaces", false,
&state.keep_spaces)) < 0
)
return code;
state.cstate = cstate;
state.binary = (is_eexec ? -1 : 1);
code = 1;
} else {
state.binary = 1;
code = eexec_param(op, &state.cstate);
}
if (code < 0)
return code;
/*
* If we're reading a .PFB file, let the filter know about it,
* so it can read recklessly to the end of the binary section.
*/
if (r_has_type(op - 1, t_file)) {
stream *s = (op - 1)->value.pfile;
if (s->state != 0 && s->state->templat == &s_PFBD_template) {
stream_PFBD_state *pss = (stream_PFBD_state *)s->state;
state.pfb_state = pss;
/*
* If we're reading the binary section of a PFB stream,
* avoid the conversion from binary to hex and back again.
*/
if (pss->record_type == 2) {
/*
* The PFB decoder may have converted some data to hex
* already. Convert it back if necessary.
*/
if (pss->binary_to_hex && sbufavailable(s) > 0) {
state.binary = 0; /* start as hex */
state.hex_left = sbufavailable(s);
} else {
state.binary = 1;
}
pss->binary_to_hex = 0;
}
}
}
return filter_read(i_ctx_p, code, &s_exD_template, (stream_state *)&state, 0);
}
int
zA85D(os_ptr op)
{ return filter_read(op, &s_A85D_procs, NULL);
}
void M2_p2o2::each_train_one_iter()
{
static bool** STA_noprobs = 0; //static ine, init only once
if(STA_noprobs==0 && !filter_read(STA_noprobs)){
//init only once
int all_tokens_train=0,all_token_filter_wrong=0;
time_t now;
time(&now);
cout << "-Preparing no_probs at " << ctime(&now) << endl;
STA_noprobs = new bool*[training_corpus->size()];
for(unsigned int i=0;i<training_corpus->size();i++){
DependencyInstance* x = training_corpus->at(i);
STA_noprobs[i] = get_cut_o1(x,mfo1,dict,hp->CONF_score_o1filter_cut);
all_tokens_train += x->length()-1;
for(int m=1;m<x->length();m++)
if(STA_noprobs[i][get_index2(x->length(),x->heads->at(m),m)])
all_token_filter_wrong ++;
}
cout << "For o1 filter: all " << all_tokens_train << ";filter wrong " << all_token_filter_wrong << endl;
filter_write(STA_noprobs);
}
//per-sentence approach
int num_sentences = training_corpus->size();
//statistics
int skip_sent_num = 0;
int all_forward_instance = 0;
int all_inst_right = 0;
int all_inst_wrong = 0;
//some useful info
int odim = mach->get_odim();
//training
time_t now;
time(&now); //ctime is not rentrant ! use ctime_r() instead if needed
cout << "##*** // Start the p2o2 training for iter " << cur_iter << " at " << ctime(&now)
<< "with lrate " << cur_lrate << endl;
cout << "#Sentences is " << num_sentences << " and resample (about)" << num_sentences*hp->CONF_NN_resample << endl;
for(int i=0;i<num_sentences;){
//random skip (instead of shuffling every time)
if(drand48() > hp->CONF_NN_resample || training_corpus->at(i)->length() >= hp->CONF_higho_toolong){
skip_sent_num ++;
i ++;
continue;
}
mach->prepare_batch();
//if nesterov update before each batch (pre-update)
if(hp->CONF_NESTEROV_MOMENTUM)
mach->nesterov_update(hp->CONF_UPDATE_WAY,hp->CONF_MOMENTUM_ALPHA);
//main batch
int this_sentence = 0;
int this_instance = 0;
for(;;){
//forward
DependencyInstance* x = training_corpus->at(i);
const int length = x->length();
nn_input* the_inputs;
REAL *fscores = forward_scores_o2sib(x,mach,&the_inputs,dict->get_helper(),0,STA_noprobs[i],hp);
this_instance += the_inputs->get_numi();
all_forward_instance += the_inputs->get_numi();
all_inst_right += the_inputs->inst_good;
all_inst_wrong += the_inputs->inst_bad;
this_sentence ++;
i++;
the_scores::Scores<REAL_SCORES>* rscores = get_the_scores(the_inputs,fscores,mach->get_odim(),the_inputs->get_numi());
REAL_SCORES* tmp_marginals = LencodeMarginals_o2sib(length,*rscores);
// //two situations
// int length = x->length();
// if(!hp->CONF_labeled){
// //calculate prob
// rscores = rearrange_scores_o2sib(x,mach,the_inputs,fscores,0,0,0,hp);
// tmp_marginals = encodeMarginals_o2sib(length,rscores);
// }
// else{
// //calculate prob
// rscores = rearrange_scores_o2sib(x,mach,the_inputs,fscores,0,0,0,hp);
// tmp_marginals = LencodeMarginals_o2sib(length,rscores,mach->get_odim());
// }
//set gradients
int HERE_dim = the_inputs->num_width;
REAL* to_assign = fscores;
for(int ii=0;ii<the_inputs->num_inst*HERE_dim;ii+=HERE_dim){
int tmph = the_inputs->inputs->at(ii);
int tmpm = the_inputs->inputs->at(ii+1);
int tmps = the_inputs->inputs->at(ii+2);
if(tmps<0)
tmps = tmph;
int tmp_goal = the_inputs->goals->at(ii/HERE_dim);
REAL_SCORES* from_mar = tmp_marginals+odim*(ii/HERE_dim);
for(int once=0;once<odim;once++,to_assign++){
if(tmp_goal == once)
*to_assign = -1 * (1 - from_mar[once]) + *to_assign * hp->CONF_score_p2reg;
else
*to_assign = from_mar[once] + *to_assign * hp->CONF_score_p2reg; //now object is maximum
}
}
//backward
mach->backward(fscores);
delete the_inputs;
delete rscores;
delete []tmp_marginals;
if(i>=num_sentences)
break;
//out of the mini-batch
while(training_corpus->at(i)->length() >= hp->CONF_higho_toolong){ //HAVE to compromise, bad choice
skip_sent_num ++;
i ++;
}
if(i>=num_sentences)
break;
if(hp->CONF_minibatch > 0){
if(this_sentence >= hp->CONF_minibatch)
break;
}
else{
if(this_instance >= -1*hp->CONF_minibatch)
break;
}
}
//real update
mach->update(hp->CONF_UPDATE_WAY,cur_lrate,hp->CONF_NN_WD,hp->CONF_MOMENTUM_ALPHA,hp->CONF_RMS_SMOOTH);
}
cout << "Iter done, skip " << skip_sent_num << " sentences and f&b " << all_forward_instance
<< ";good/bad: " << all_inst_right << "/" << all_inst_wrong << endl;
}
