bool Z3DNetworkEvaluator::deinitializeNetwork()
{
if (m_locked) {
LWARN() << "locked.";
return false;
}
if (!m_canvasRenderer) {
LWARN() << "no network.";
return false;
}
lock();
bool failed = false;
for (size_t i = 0; i < m_renderingOrder.size(); ++i) {
Z3DProcessor* processor = m_renderingOrder[i];
if (processor->isInitialized()) {
try {
getGLFocus();
processor->deinitialize();
processor->m_initialized = false;
CHECK_GL_ERROR;
}
catch (const Exception& e) {
LERROR() << "Failed to deinitialize processor" << processor->getClassName() << e.what();
// don't break, try to deinitialize the other processors even if one failed
failed = true;
}
}
}
unlock();
return !failed;
}
void Z3DCheckOpenGLStateProcessWrapper::warn(const Z3DProcessor* p, const QString& message)
{
if (p) {
LWARN() << "Invalid OpenGL state after processing" << p->getClassName() << ":" << message;
}
else {
LWARN() << "Invalid OpenGL state before network processing:" << message;
}
}
bool Z3DCanvasRenderer::renderToImage(const QString &filename, int width, int height, Z3DScreenShotType sst)
{
if (!m_canvas) {
LWARN() << "no canvas assigned";
m_renderToImageError = "No canvas assigned";
return false;
}
glm::ivec2 oldDimensions = m_inport.getSize();
// resize texture container to desired image dimensions and propagate change
m_canvas->getGLFocus();
m_inport.setExpectedSize(glm::ivec2(width, height));
m_leftEyeInport.setExpectedSize(glm::ivec2(width, height));
m_rightEyeInport.setExpectedSize(glm::ivec2(width, height));
emit requestUpstreamSizeChange(this);
// render with adjusted viewport size
bool success = renderToImage(filename, sst);
// reset texture container dimensions from canvas size
m_inport.setExpectedSize(oldDimensions);
m_leftEyeInport.setExpectedSize(oldDimensions);
m_rightEyeInport.setExpectedSize(oldDimensions);
emit requestUpstreamSizeChange(this);
return success;
}
int realloc_ring_buffer(BufferPtr *buffer, BufferPtr *data, int new_size) {
char *new_buffer = 0;
int tail_size, data_size = (data->end > data->data) ?
BPT_SIZE(data) : (BPT_SIZE(buffer) - BPT_SIZE(data));
if (new_size < data_size) {
LWARN("desired buffer size (%d) is smaller than the actual data size (%d), expanding",
new_size, data_size);
new_size = data_size;
}
new_buffer = malloc(new_size);
if (!new_buffer) {
LERROR("out of memory when allocating a buffer of size %d", new_size);
return BPT_SIZE(buffer);
}
if (data->data != data->end) {
if (data->data > data->end) {
tail_size = buffer->end - data->data;
memcpy(new_buffer, data->data, tail_size);
memcpy(new_buffer + tail_size, buffer->data, data->end - buffer->data);
} else {
memcpy(new_buffer, data->data, BPT_SIZE(data));
}
}
free(buffer->data);
data->data = buffer->data = new_buffer;
buffer->end = new_buffer + new_size;
data->end = new_buffer + data_size;
return new_size;
}
bool PresentationEngineManager::startDocument( const std::string &file ) {
LDEBUG( "PresentationEngineManager", "Start document: doc=%s", file.c_str() );
if (formatter()) {
_sys->enqueue( boost::bind( &PresentationEngineManager::stopDocument, this ) );
{ // Wait for compare
boost::unique_lock<boost::mutex> lock( _stopMutex );
while (!_stopped) {
_stopWakeup.wait( lock );
}
_stopped = false;
}
}
// Init settings module
player::settings::init();
initFormatter( file );
if (!formatter()->parseDocument()) {
LWARN( "PresentationEngineManager", "parseDocument fail" );
return false;
}
// Play formatter
_sys->enqueue( boost::bind( &FormatterMediator::play, formatter() ) );
return true;
}
void Z3DShaderProgram::bindVolume(const QString &name, Z3DVolume *volume,
GLint minFilter, GLint magFilter)
{
if (!volume)
return;
m_textureUnitManager.nextAvailableUnit();
m_textureUnitManager.activateCurrentUnit();
if (!volume->getTexture()) {
LWARN() << "volume do not contains any texture";
glActiveTexture(GL_TEXTURE0);
return;
}
volume->getTexture()->bind();
GLenum target = volume->getTexture()->getTextureTarget();
// texture filtering
glTexParameteri(target, GL_TEXTURE_MAG_FILTER, magFilter);
glTexParameteri(target, GL_TEXTURE_MIN_FILTER, minFilter);
volume->setUniform(*this, name, m_textureUnitManager.getCurrentUnitNumber());
glActiveTexture(GL_TEXTURE0);
CHECK_GL_ERROR;
}
static GC* gc_unique_decide_collection_algo(char* unique_algo, Boolean has_los)
{
/* if unique_algo is not set, gc_gen_decide_collection_algo is called. */
assert(unique_algo);
GC_PROP = ALGO_POOL_SHARE | ALGO_DEPTH_FIRST | ALGO_IS_UNIQUE;
assert(!has_los); /* currently unique GCs don't use LOS */
if(has_los)
GC_PROP |= ALGO_HAS_LOS;
Boolean use_default = FALSE;
GC* gc;
string_to_upper(unique_algo);
if(!strcmp(unique_algo, "MOVE_COMPACT")){
GC_PROP |= ALGO_COMPACT_MOVE;
gc = gc_mc_create();
}else if(!strcmp(unique_algo, "MARK_SWEEP")){
GC_PROP |= ALGO_MS_NORMAL;
gc = gc_ms_create();
}else{
LWARN(48, "\nGC algorithm setting incorrect. Will use default value.\n");
GC_PROP |= ALGO_COMPACT_MOVE;
gc = gc_mc_create();
}
return gc;
}
void recv_command(struct ev_loop *loop, struct ev_io *io, int rev) {
ADEBUG(!(rev & EV_ERROR));
void *sock = SHIFT(io, config_zmqsocket_t, _watch)->_sock;
while(TRUE) {
Z_SEQ_INIT(msg, sock);
Z_RECV_START(msg, break);
size_t len;
while(TRUE) {
len = zmq_msg_size(&msg);
REPLY_COMMAND(sock, msg, TRUE);
if(!len) break;
Z_RECV_NEXT(msg);
}
Z_RECV(msg);
len = zmq_msg_size(&msg);
char *data = zmq_msg_data(&msg);
LDEBUG("Got command `%.*s`", len, data);
if COMMAND(list_commands) {
// Keep alphabetically sorted
REPLY_SHORT(sock, msg, "list_commands", TRUE);
REPLY_SHORT(sock, msg, "pause_websockets", TRUE);
REPLY_SHORT(sock, msg, "resume_websockets", TRUE);
REPLY_SHORT(sock, msg, "sync_now", TRUE);
REPLY_SHORT(sock, msg, "reopen_logs", FALSE);
} else if COMMAND(pause_websockets) {
LWARN("Pausing websockets because of command");
pause_websockets(TRUE);
REPLY_SHORT(sock, msg, "paused", FALSE);
} else if COMMAND(resume_websockets) {
bool Z3DCanvasRenderer::renderToImage(const QString &filename, Z3DScreenShotType sst)
{
if (!m_canvas) {
LWARN() << "no canvas assigned";
m_renderToImageError = "No canvas assigned";
return false;
}
// enable render-to-file on next process
m_renderToImageFilename = filename;
m_renderToImage = true;
m_renderToImageError.clear();
m_renderToImageType = sst;
// force rendering pass
if (m_canvas->format().stereo() && sst == MonoView) {
LERROR() << "impossible configuration";
assert(false);
}
if (!m_canvas->format().stereo() && sst != MonoView)
m_canvas->setFakeStereoOnce();
m_canvas->forceUpdate();
return (m_renderToImageError.isEmpty());
}
void class_throw_linking_error(Class_Handle ch, unsigned index, unsigned opcode)
{
ASSERT_RAISE_AREA;
tmn_suspend_enable();
ConstantPool& cp = ch->get_constant_pool();
if(cp.is_entry_in_error(index)) {
exn_raise_object(cp.get_error_cause(index));
tmn_suspend_disable();
return; // will return in interpreter mode
}
switch(opcode) {
case OPCODE_NEW:
class_can_instantiate(cp.get_class_class(index), LINK_THROW_ERRORS);
break;
case OPCODE_PUTFIELD:
field_can_link(ch, cp.get_ref_field(index),
CAN_LINK_FROM_FIELD, LINK_WRITE_ACCESS, LINK_THROW_ERRORS);
break;
case OPCODE_GETFIELD:
field_can_link(ch, cp.get_ref_field(index),
CAN_LINK_FROM_FIELD, LINK_READ_ACCESS, LINK_THROW_ERRORS);
break;
case OPCODE_PUTSTATIC:
field_can_link(ch, cp.get_ref_field(index),
CAN_LINK_FROM_STATIC, LINK_WRITE_ACCESS, LINK_THROW_ERRORS);
break;
case OPCODE_GETSTATIC:
field_can_link(ch, cp.get_ref_field(index),
CAN_LINK_FROM_STATIC, LINK_READ_ACCESS, LINK_THROW_ERRORS);
break;
case OPCODE_INVOKEINTERFACE:
method_can_link_interface(ch, index, cp.get_ref_method(index),
LINK_THROW_ERRORS);
break;
case OPCODE_INVOKESPECIAL:
method_can_link_special(ch, index, cp.get_ref_method(index),
LINK_THROW_ERRORS);
break;
case OPCODE_INVOKESTATIC:
method_can_link_static(ch, index, cp.get_ref_method(index),
LINK_THROW_ERRORS);
break;
case OPCODE_INVOKEVIRTUAL:
method_can_link_virtual(ch, index, cp.get_ref_method(index),
LINK_THROW_ERRORS);
break;
default:
// FIXME Potentially this can be any RuntimeException or Error
// The most probable case is OutOfMemoryError.
LWARN(5, "**Java exception occured during resolution under compilation");
exn_raise_object(VM_Global_State::loader_env->java_lang_OutOfMemoryError);
//ASSERT(0, "Unexpected opcode: " << opcode);
break;
}
tmn_suspend_disable();
}
BOOL config_read() {
char cfg_opt_name[CONFIG_OPTION_LENGTH];
int error;
FILE *file = 0;
LINFO("loading configuration from '%s'", config.filename[0] ? config.filename : "stdin");
if (config.filename[0]) {
file = fopen(config.filename, "r");
if (!file) {
error = errno;
LERRNO("could not open configuration file %s for reading",
error, config.filename);
return FALSE;
}
} else {
file = stdin;
rewind(file);
}
while (fscanf(file, "%1023s", cfg_opt_name) == 1) {
if (cfg_opt_name[0] == '#') {
// Skip comments
fscanf(file, "%*[^\n]\n");
continue;
}
READ_CONFIG_STR(network.host)
READ_CONFIG_INT(network.port)
READ_CONFIG_STR(component.password)
READ_CONFIG_STR(component.hostname)
READ_CONFIG_INT(reader.buffer)
READ_CONFIG_INT(reader.block)
READ_CONFIG_INT(reader.queue)
READ_CONFIG_INT(writer.buffer)
READ_CONFIG_STR(worker.data_file)
READ_CONFIG_INT(worker.threads)
READ_CONFIG_INT(worker.buffer)
READ_CONFIG_INT(worker.deciseconds_limit)
READ_CONFIG_STR(acl.data_file)
READ_CONFIG_INT(acl.default_role)
READ_CONFIG_INT(logger.level)
LWARN("unknown config option '%s'", cfg_opt_name);
}
if (file != stdin) {
fclose(file);
}
return TRUE;
}
Class* Global_Env::LoadCoreClass(const String* s)
{
Class* clss =
bootstrap_class_loader->LoadVerifyAndPrepareClass(this, s);
if(clss == NULL) {
// print error diagnostics and exit VM
LWARN(4, "Failed to load bootstrap class {0}" << s->bytes);
log_exit(1);
}
return clss;
}
void Z3DRenderOutputPort::resize(const glm::ivec2 &newsize)
{
if (m_size == newsize)
return;
if (newsize == glm::ivec2(0)) {
LWARN() << "invalid size:" << newsize;
return;
}
if (newsize.x > Z3DGpuInfoInstance.getMaxTextureSize() ||
newsize.y > Z3DGpuInfoInstance.getMaxTextureSize()) {
LWARN() << "size" << newsize << "exceeds texture size limit:"
<< Z3DGpuInfoInstance.getMaxTextureSize();
return;
}
if (m_renderTarget) {
m_renderTarget->resize(newsize);
}
m_resultIsValid = false;
m_size = newsize;
}
ZColorMapWidgetWithEditorWindow::ZColorMapWidgetWithEditorWindow(ZColorMapParameter *cm, QMainWindow *minWin, QWidget *parent) :
ZClickableColorMapLabel(cm, parent), m_colorMap(cm), m_mainWin(minWin), m_editorWindow(NULL)
{
if (!m_mainWin) {
LWARN() << "If Main Window is empty, this widget will show nothing when clicked.";
} else {
connect(m_mainWin, SIGNAL(destroyed()), this, SLOT(mainWindowDestroyed()));
}
// if (m_mainWin) {
// connect(m_mainWin, SIGNAL(closed()), this, SLOT(mainWindowClosed()));
// }
}
void Z3DCamera::updateCamera()
{
m_viewVector = glm::normalize(m_center - m_eye);
m_centerDist = glm::length(m_center - m_eye);
// make sure upVector is not parallel to viewVector
if (std::abs(glm::dot(m_upVector, m_viewVector)) >= 0.9) {
LWARN() << "Resetting view up since view plane normal is parallel";
m_upVector = glm::cross(m_viewVector, glm::vec3(1.f, 0.f, 0.f));
if (glm::dot(m_upVector,m_upVector) < 0.001)
m_upVector = glm::cross(m_viewVector, glm::vec3(0.f, 1.f, 0.f));
m_upVector = glm::normalize(m_upVector);
}
m_strafeVector = glm::cross(m_viewVector, m_upVector);
m_eyeSeparation = 2.f * m_centerDist * std::tan(m_eyeSeparationAngle/2.f);
}
void openlogs() {
if(logconfig->error) {
int oldlogfile = logfile;
logfile = open(logconfig->error, O_APPEND|O_WRONLY|O_CREAT, 0666);
if(logfile <= 0) {
logfile = oldlogfile;
LWARN("Can't open logfile. Continuing writing to stdout");
} else {
LINFO("Log sucessfully opened");
setcloexec(logfile);
dup2(logfile, STDOUT);
dup2(logfile, STDERR);
}
}
}
///functor: Run composition itself, separate from load times (e.g. which may include an fst expansion).
VectorFst<Arc> * operator() () {
if (!fst_.NumStates() ) {
LWARN ("Empty lattice. ... Skipping LM application!");
return NULL;
}
while ( qc_.size() ) {
StateId s = qc_.front();
qc_.pop();
pair<StateId, const typename KenLMModelT::State> p = get ( s );
StateId& s1 = p.first;
const typename KenLMModelT::State s2 = p.second;
for ( ArcIterator< VectorFst<Arc> > arc1 ( fst_, s1 ); !arc1.Done();
arc1.Next() ) {
const Arc& a1 = arc1.Value();
float w = 0;
float wp = wp_;
typename KenLMModelT::State nextlmstate;
if ( epsilons_.find ( a1.olabel ) == epsilons_.end() ) {
w = lmmodel_.Score ( s2, idbridge_.map (a1.olabel), nextlmstate ) * natlog10_;
//silly hack
if ( a1.olabel <= 2 ) {
wp = 0;
if (a1.olabel == 1 ) w = 0; //get same result as srilm
}
} else {
nextlmstate = s2;
wp = 0; //We don't count epsilon labels
}
pair<StateId, bool> nextp = add ( nextlmstate
, a1.nextstate
, fst_.Final ( a1.nextstate ) );
StateId& newstate = nextp.first;
bool visited = nextp.second;
composed_->AddArc ( s
, Arc ( a1.ilabel, a1.olabel
, Times ( a1.weight, Times (mw_ ( w ) , mw_ (wp) ) )
, newstate ) );
//Finally, only add newstate to the queue if it hasn't been visited previously
if ( !visited ) {
qc_.push ( newstate );
}
}
}
LINFO ( "Done! Number of states=" << composed_->NumStates() );
return composed_;
};
Z3DTransferFunctionWidgetWithEditorWindow::Z3DTransferFunctionWidgetWithEditorWindow(Z3DTransferFunctionParameter *tf,
QMainWindow *minWin, QWidget *parent)
: ZClickableTransferFunctionLabel(tf, parent)
, m_transferFunction(tf)
, m_mainWin(minWin)
, m_editorWindow(NULL)
{
if (!m_mainWin) {
LWARN() << "If Main Window is empty, this widget will show nothing when clicked.";
} else {
connect(m_mainWin, SIGNAL(destroyed()), this, SLOT(mainWindowDestroyed()));
}
// if (m_mainWin) {
// connect(m_mainWin, SIGNAL(closed()), this, SLOT(mainWindowClosed()));
// }
}
void reopenlogs() {
if(logconfig->error) {
int oldlogfile = logfile;
logfile = open(logconfig->error, O_APPEND|O_WRONLY|O_CREAT, 0666);
if(logfile <= 0) {
logfile = oldlogfile;
SWARN2("Can't open logfile. Continuing writing old file");
} else {
LINFO("New log file successfully opened");
setcloexec(logfile);
dup2(logfile, STDOUT);
dup2(logfile, STDERR);
close(oldlogfile);
}
} else {
LWARN("No logfile specified");
}
}
void Z3DShaderProgram::bindVolume(const QString &name, Z3DVolume *volume)
{
if (!volume)
return;
m_textureUnitManager.nextAvailableUnit();
m_textureUnitManager.activateCurrentUnit();
if (!volume->getTexture()) {
LWARN() << "volume do not contains any texture";
glActiveTexture(GL_TEXTURE0);
return;
}
volume->getTexture()->bind();
volume->setUniform(*this, name, m_textureUnitManager.getCurrentUnitNumber());
glActiveTexture(GL_TEXTURE0);
CHECK_GL_ERROR;
}
void ZDvidTarget::setFromSourceString(const std::string &sourceString)
{
clear();
std::vector<std::string> tokens = ZString(sourceString).tokenize(':');
m_node.setFromSourceToken(tokens);
if (tokens.size() >= 5) {
setBodyLabelName(tokens[4]);
}
if (tokens.size() >= 6) {
setGrayScaleName(tokens[5]);
}
#if 0
if (tokens.size() < 4 || tokens[0] != "http" || tokens[0] != "mock") {
#if defined(_QT_APPLICATION_)
LWARN() << "Invalid source string for dvid target:" << sourceString.c_str();
#else
RECORD_WARNING_UNCOND("Invalid source string");
#endif
} else {
int port = -1;
if (!tokens[2].empty()) {
port = ZString::FirstInteger(tokens[2]);
if (tokens[2][0] == '-') {
port = -port;
}
}
set(tokens[1], tokens[3], port);
if (tokens.size() >= 5) {
setBodyLabelName(tokens[4]);
}
if (tokens.size() >= 6) {
setGrayScaleName(tokens[5]);
}
if (tokens[0] == "mock") {
setMock(true);
}
}
#endif
}
/**
* Constructor. Initializes on-the-fly composition with a language model.
* \param fst Machine you want to apply the language to. Pass a delayed machine if you can, as it will expand it in constructor.
* \param model A KenLM language model
* \param epsilons List of words to work as epsilons
* \param natlog Use or not natural logs
* \param lmscale Language model scale
*/
ApplyLanguageModelOnTheFly ( const Fst<Arc>& fst,
KenLMModelT& model,
#ifndef USE_GOOGLE_SPARSE_HASH
unordered_set<Label>& epsilons,
#else
google::dense_hash_set<Label>& epsilons,
#endif
bool natlog,
float lmscale ,
float lmwp,
const IdBridgeT& idbridge
) :
composed_ ( NULL ) ,
natlog10_ ( natlog ? -lmscale* ::log ( 10.0 ) : -lmscale ),
fst_ ( fst ),
lmmodel_ ( model ),
vocab_ ( model.GetVocabulary() ),
wp_ ( lmwp ) ,
epsilons_ ( epsilons ) ,
history ( model.Order(), 0),
idbridge_ (idbridge) {
#ifdef USE_GOOGLE_SPARSE_HASH
stateexistence_.set_empty_key ( numeric_limits<ull>::max() );
statemap_.set_empty_key ( numeric_limits<uint64_t>::max() );
basic_string<unsigned> aux (KENLM_MAX_ORDER, numeric_limits<unsigned>::max() );
seenlmstates_.set_empty_key ( aux );
#endif
buffersize = ( model.Order() - 1 ) * sizeof ( unsigned int );
buffer = const_cast<unsigned *> ( history.c_str() );
if (!fst_.NumStates() ) {
LWARN ("Empty lattice");
return;
}
composed_ = new VectorFst<Arc>;
typename KenLMModelT::State bs = model.NullContextState();
///Initialize with first state
pair<StateId, bool> nextp = add ( bs, fst_.Start(),
fst_.Final ( fst_.Start() ) );
qc_.push ( nextp.first );
composed_->SetStart ( nextp.first );
};
void ZDvidTarget::setFromSourceString(
const std::string &sourceString, ZDvid::EDataType dataType)
{
clear();
std::vector<std::string> tokens = ZString(sourceString).tokenize(':');
if (tokens.size() < 4 || tokens[0] != "http" || tokens[0] != "mock") {
#if defined(_QT_APPLICATION_)
LWARN() << "Invalid source string for dvid target:" << sourceString.c_str();
#else
RECORD_WARNING_UNCOND("Invalid source string");
#endif
} else {
int port = -1;
if (!tokens[2].empty()) {
port = ZString::FirstInteger(tokens[2]);
if (tokens[2][0] == '-') {
port = -port;
}
}
set(tokens[1], tokens[3], port);
if (tokens.size() >= 5) {
switch (dataType) {
case ZDvid::TYPE_LABELVOL:
setBodyLabelName(tokens[4]);
break;
case ZDvid::TYPE_UINT8BLK:
setGrayScaleName(tokens[4]);
break;
default:
break;
}
}
if (tokens[0] == "mock") {
setMock(true);
}
}
}
bool PresentationEngineManager::run() {
if (!_sys->canRun()) {
LWARN( "PresentationEngineManager", "no device configurated" );
return false;
}
onStart();
{ // Add listener for keys
::util::key::Keys keys;
keys.insert( ::util::key::greater_than_sign );
keys.insert( ::util::key::less_than_sign );
keys.insert( ::util::key::f10 );
keys.insert( ::util::key::power );
keys.insert( ::util::key::escape );
keys.insert( ::util::key::stop );
keys.insert( ::util::key::exit );
keys.insert( ::util::key::f12 );
keys.insert( ::util::key::pause );
player::input::InputCallback fnc = boost::bind( &PresentationEngineManager::userEventReceived, this, _1, _2 );
_sys->addInputListener( this, fnc, keys );
}
// Run canvas loop
_sys->run();
stopDocument();
onStop();
// Remove listeners
_sys->delInputListener( this );
return true;
}
/**
* \brief Runs the lattice building procedure.
* \param d Contains the data structure with all the necessary elements (i.e. cykdata) and in which will be store a pointer to the
* output lattice.
*/
bool run ( Data& d ) {
cykfstresult_.DeleteStates();
this->d_ = &d;
hieroindexexistence_.clear();
LINFO ( "Running HiFST" );
//Reset one-time warnings for inexistent language models.
warned_ = false;
pdtparens_.clear();
cykdata_ = d.cykdata;
if ( !USER_CHECK ( cykdata_, "cyk parse has not been executed previously?" ) ) {
resetExternalData (d);
return true;
}
if ( d.cykdata->success == CYK_RETURN_FAILURE ) {
///Keep calm, return empty lattice and carry on
fst::VectorFst<Arc> aux;
d.fsts[outputkey_] = &cykfstresult_;
d.vcat = cykdata_->vcat;
resetExternalData (d);
return false;
}
///If not yet, initialize now functor with local conditions.
initLocalConditions();
rtn_ = new RTNT;
if ( localprune_ )
rtnnumstates_ = new ExpandedNumStatesRTNT;
rfba_ = new ReplaceFstByArcT ( cykdata_->vcat, replacefstbyarc_,
replacefstbyarcexceptions_, aligner_, replacefstbynumstates_ );
piscount_ = 0; //reset pruning-in-search count to 0
LINFO ( "Second Pass: FST-building!" );
d.stats->setTimeStart ( "lattice-construction" );
//Owned by rtn_;
fst::Fst<Arc> *sfst = buildRTN ( cykdata_->categories["S"], 0,
cykdata_->sentence.size() - 1 );
d.stats->setTimeEnd ( "lattice-construction" );
cykfstresult_ = (*sfst);
LINFO ( "Final - RTN head optimizations !" );
optimize ( &cykfstresult_ ,
std::numeric_limits<unsigned>::max() ,
!hipdtmode_ && optimize_
);
FORCELINFO ("Stats for Sentence " << d.sidx <<
": local pruning, number of times=" << piscount_);
d.stats->lpcount = piscount_; //store local pruning counts in stats
LINFO ("RTN expansion starts now!");
//Expand...
{
///Define hieroindex
Label hieroindex = APBASETAG + 1 * APCCTAG + 0 * APXTAG +
( cykdata_->sentence.size() - 1 ) * APYTAG;
if ( hieroindexexistence_.find ( hieroindex ) == hieroindexexistence_.end() )
pairlabelfsts_.push_back ( pair< Label, const fst::Fst<Arc> * > ( hieroindex,
&cykfstresult_ ) );
///Optimizations over the rtn -- they are generally worth doing...
fst::ReplaceUtil<Arc> replace_util (pairlabelfsts_, hieroindex,
!aligner_); //has ownership of modified rtn fsts
if (rtnopt_) {
LINFO ("rtn optimizations...");
d_->stats->setTimeStart ("replace-opts");
replace_util.ReplaceTrivial();
replace_util.ReplaceUnique();
replace_util.Connect();
pairlabelfsts_.clear();
replace_util.GetFstPairs (&pairlabelfsts_);
d_->stats->setTimeEnd ("replace-opts");
}
//After optimizations, we can write RTN if required by user
writeRTN();
boost::scoped_ptr< fst::VectorFst<Arc> > efst (new fst::VectorFst<Arc>);
if (!hipdtmode_ ) {
LINFO ("Final Replace (RTN->FSA), main index=" << hieroindex);
d_->stats->setTimeStart ("replace-rtn-final");
Replace (pairlabelfsts_, &*efst, hieroindex, !aligner_);
d_->stats->setTimeEnd ("replace-rtn-final");
} else {
LINFO ("Final Replace (RTN->PDA)");
d_->stats->setTimeStart ("replace-pdt-final");
Replace (pairlabelfsts_, &*efst, &pdtparens_, hieroindex);
d_->stats->setTimeEnd ("replace-pdt-final");
LINFO ("Number of pdtparens=" << pdtparens_.size() );
}
LINFO ("Removing Epsilons...");
fst::RmEpsilon<Arc> ( &*efst );
LINFO ("Done! NS=" << efst->NumStates() );
//Apply filters
applyFilters ( &*efst );
//Compose with full reference lattice to ensure that final lattice is correct.
if ( d.fsts.find ( fullreferencelatticekey_ ) != d.fsts.end() ) {
if ( static_cast< fst::VectorFst<Arc> * >
(d.fsts[fullreferencelatticekey_])->NumStates() > 0 ) {
LINFO ( "Composing with full reference lattice, NS=" <<
static_cast< fst::VectorFst<Arc> * >
(d.fsts[fullreferencelatticekey_])->NumStates() );
fst::Compose<Arc> ( *efst,
* ( static_cast<fst::VectorFst<Arc> * > (d.fsts[fullreferencelatticekey_]) ),
&*efst );
LINFO ( "After composition: NS=" << efst->NumStates() );
} else {
LINFO ( "No composition with full ref lattice" );
};
} else {
LINFO ( "No composition with full ref lattice" );
};
//Apply language model
fst::VectorFst<Arc> *res = NULL;
if (efst->NumStates() )
res = applyLanguageModel ( *efst );
else {
LWARN ("Empty lattice -- skipping LM application");
}
if ( res != NULL ) {
boost::shared_ptr<fst::VectorFst<Arc> >latlm ( res );
if ( latlm.get() == efst.get() ) {
LWARN ( "Yikes! Unexpected situation! Will it crash? (muhahaha) " );
}
//Todo: union with shortest path...
if ( pruneweight_ < std::numeric_limits<float>::max() ) {
if (!hipdtmode_ || pdtparens_.empty() ) {
LINFO ("Pruning, weight=" << pruneweight_);
fst::Prune<Arc> (*latlm, &cykfstresult_, mw_ ( pruneweight_ ) );
} else {
LINFO ("Expanding, weight=" << pruneweight_);
fst::ExpandOptions<Arc> eopts (true, false, mw_ ( pruneweight_ ) );
Expand ( *latlm, pdtparens_, &cykfstresult_, eopts);
pdtparens_.clear();
}
} else {
LINFO ("Copying through full lattice with lm scores");
cykfstresult_ = *latlm;
}
} else {
LINFO ("Copying through full lattice (no lm)");
cykfstresult_ = *efst;
}
if ( hieroindexexistence_.find ( hieroindex ) == hieroindexexistence_.end() )
pairlabelfsts_.pop_back();
}
pairlabelfsts_.clear();
LINFO ( "Reps" );
fst::RmEpsilon ( &cykfstresult_ );
LINFO ( "NS=" << cykfstresult_.NumStates() );
//This should delete all pertinent fsas...
LINFO ( "deleting data stuff..." );
delete rtn_;
if ( localprune_ )
delete rtnnumstates_;
delete rfba_;
d.vcat = cykdata_->vcat;
resetExternalData (d);
d.fsts[outputkey_] = &cykfstresult_;
if (hipdtmode_ && pdtparens_.size() )
d.fsts[outputkey_ + ".parens" ] = &pdtparens_;
LINFO ( "done..." );
FORCELINFO ( "End Sentence ******************************************************" );
d.stats->setTimeEnd ( "sent-dec" );
d.stats->message += "[" + ucam::util::getTimestamp() + "] End Sentence\n";
return false;
};
inline void init_param_options ( int argc, const char* argv[],
po::variables_map *vm ) {
try {
po::options_description desc ( "Command-line/configuration file options" );
initAllCreateSSGrammarOptions (desc); // All createssgrammar options are used
initCommonApplylmOptions (desc); // Add generic language model options
desc.add_options()
( HifstConstants::kServerEnable.c_str()
, po::value<std::string>()->default_value ( "no" )
, "Run in server mode (yes|no)" )
( HifstConstants::kServerPort.c_str()
, po::value<short>()->default_value ( 1209 )
, "Server port" )
( HifstConstants::kTargetStore.c_str()
, po::value<std::string>()->default_value ( "-" )
, "Source text file -- this option is ignored in server mode" )
( HifstConstants::kFeatureweights.c_str()
, po::value<std::string>()->default_value ( "" )
, "Feature weights applied in hifst. This is a comma-separated sequence "
"of language model(s) and grammar feature weights.\n"
"IMPORTANT: If this option is not empty string, then it will override "
"any values in lm.featureweights and grammar.featureweights"
)
( HifstConstants::kReferencefilterLoad.c_str()
, po::value<std::string>()->default_value ( "" )
, "Reference lattice to filter the translation" )
( HifstConstants::kReferencefilterWrite.c_str()
, po::value<std::string>()->default_value ( "" )
, "Write reference lattice" )
( HifstConstants::kReferencefilterSubstring.c_str()
, po::value<std::string>()->default_value ( "yes" )
, "Substring the reference lattice (yes|no)" )
( HifstConstants::kReferencefilterPrunereferenceweight.c_str()
, po::value<float>()->default_value ( std::numeric_limits<float>::max() )
, "Likelihood beam to prune the reference lattice" )
( HifstConstants::kReferencefilterPrunereferenceshortestpath.c_str()
, po::value<unsigned>()->default_value (
std::numeric_limits<unsigned>::max() ) )
( HifstConstants::kCykparserHrmaxheight.c_str()
, po::value<unsigned>()->default_value ( 10 )
, "Default maximum span for hierarchical rules" )
( HifstConstants::kCykparserHmax.c_str()
, po::value<std::string>()->default_value ( "" )
, "Maximum span for individual non-terminals, constrained to hrmaxheight : e.g. X,10,V,6" )
( HifstConstants::kCykparserHmin.c_str()
, po::value<std::string>()->default_value ( "" )
, "Minimum span for individual non-terminals, constrained to hrmaxheight: e.g. X,3,V,2" )
( HifstConstants::kCykparserNtexceptionsmaxspan.c_str()
, po::value<std::string>()->default_value ( "S" )
, "List of non-terminals not affected by cykparser.hrmaxheight. S should always be in this list!" )
( HifstConstants::kHifstLatticeStore.c_str()
, po::value<std::string>()->default_value ( "" )
, "Store hifst translation lattice" )
( HifstConstants::kHifstAlilatsmode.c_str()
, po::value<std::string>()->default_value ( "no" )
, "Include derivations in the left side of transducers (yes|no)" )
( HifstConstants::kHifstUsepdt.c_str()
, po::value<std::string>()->default_value ( "no" )
, "Run hifst using pdt representation, aka hipdt (yes|no)" )
( HifstConstants::kHifstRtnopt.c_str()
, po::value<std::string>()->default_value ( "yes" )
, " Use openfst rtn optimizations (yes|no)" )
( HifstConstants::kHifstOptimizecells.c_str()
, po::value<std::string>()->default_value ( "yes" )
, "Determinize/minimize any FSA component of the RTN (yes|no)" )
( HifstConstants::kHifstReplacefstbyarcNonterminals.c_str()
, po::value<std::string>()->default_value ( "" )
, "Determine which cell fsts are always replaced by single arc according to its non-terminals, e.g: replacefstbyarc=X,V" )
( HifstConstants::kHifstReplacefstbyarcNumstates.c_str()
, po::value<unsigned>()->default_value ( 4 )
, "Determine the minimum number of states that triggers replacement by arc." )
( HifstConstants::kHifstReplacefstbyarcExceptions.c_str()
, po::value<std::string>()->default_value ( "S" )
, "Categories that will definitely not be replaced (takes over replacefstbyarc and replacefstbyarc.numstates)" )
( HifstConstants::kHifstLocalpruneEnable.c_str()
, po::value<std::string>()->default_value ( "no" )
, "Apply local pruning strategy based con cyk cells and number of states (yes|no)" )
( HifstConstants::kHifstLocalpruneLmLoad.c_str()
, po::value<std::string>()->default_value ( "" )
, "Load one or more language model files: (gzipped) arpa format or kenlm binary format (uses memory mapping); separated by commas" )
( HifstConstants::kHifstLocalpruneLmFeatureweights.c_str()
, po::value<std::string>()->default_value ( "1.0" )
, "Scaling factor(s) applied to the language model: arpa_weight * -log(10) * gscale. Scales separated by commas." )
( HifstConstants::kHifstLocalpruneLmWordpenalty.c_str()
, po::value<std::string>()->default_value ( "0.0" )
, "Word penalty applied along the language models (separated by commas). Assumed as 0 if not specified " )
( HifstConstants::kHifstLocalpruneNumstates.c_str()
, po::value<unsigned>()->default_value ( 10000 )
, "Maximum number of states threshold after cell pruning an FSA, If beneath the threshold, determinization/minimization is applied to pruned lattice. Also applicable in alignment mode when filtering against substring acceptor. ")
( HifstConstants::kHifstLocalpruneConditions.c_str()
, po::value<std::string>()->default_value ( "" )
, "Local pruning conditions. These are sequences of 4-tuples separated by commas: category,span,number_of_states,weight. The three first are actual thresholds that trigger local pruning, whereas the weight is the likelihood beam for pruning, IF a language model has been applied." )
( HifstConstants::kHifstPrune.c_str()
, po::value<float>()->default_value ( std::numeric_limits<float>::max() )
, "Likelihood beam to prune the translation lattice. Only applied IF a language model is available." )
( HifstConstants::kHifstWritertn.c_str()
, po::value<std::string>()->default_value ( "")
, "Write the rtn to disk -- long list of FSAs. Use %%rtn_label%% and ? to format file names appropriately, e.g. --hifst.writertn=rtn/?/%%rtn_label%%.fst" )
( HifstConstants::kRecaserLmLoad.c_str()
, po::value<std::string>()->default_value ( "" )
, "Language model for recasing" )
( HifstConstants::kRecaserLmFeatureweight.c_str()
, po::value<std::string>()->default_value ( "1.0" )
, "Scaling factor applied to the language model" )
( HifstConstants::kRecaserUnimapLoad.c_str()
, po::value<std::string>()->default_value ( "" )
, "unigram transduction model " )
( HifstConstants::kRecaserUnimapWeight.c_str()
, po::value<float>()->default_value ( 1.0f )
, "Scaling factors applied to the unigram model " )
( HifstConstants::kRecaserPrune.c_str()
, po::value<std::string>()->default_value ( "byshortestpath,1" )
, "Choose between byshortestpath,numpaths or byweight,weight" )
( HifstConstants::kRecaserOutput.c_str()
, po::value<std::string>()->default_value ("")
, "Output true cased lattice" )
( HifstConstants::kPostproWordmapLoad.c_str()
, po::value<std::string>()->default_value ( "" )
, "Load a reverse integer mapping file so the decoder can map integers to target words" )
( HifstConstants::kPostproDetokenizeEnable.c_str()
, po::value<std::string>()->default_value ( "no" )
, "Detokenize translated 1best (yes|no) -- NOT IMPLEMENTED!" )
( HifstConstants::kPostproDetokenizeLanguage.c_str()
, po::value<std::string>()->default_value ( "" ), "NOT IMPLEMENTED" )
( HifstConstants::kPostproCapitalizefirstwordEnable.c_str()
, po::value<std::string>()->default_value ( "no" )
, "Capitalize first word (yes|no). Only applies if previously mapped back to words (postpro.wordmap.load)" )
( HifstConstants::kStatsHifstWrite.c_str()
, po::value<std::string>()->default_value ( "" )
, "Dump hifst-specific stats (cyk, local pruning, etc)" )
( HifstConstants::kStatsHifstCykgridEnable.c_str()
, po::value<std::string>()->default_value ( "no" )
, "Write cyk/rtn stats to the file (yes|no)" )
( HifstConstants::kStatsHifstCykgridCellwidth.c_str()
, po::value<unsigned>()->default_value ( 30 )
, "Width of the printed cyk cell" )
( HifstConstants::kStatsWrite.c_str()
, po::value<std::string>()->default_value ( "" )
, "Dump general stats (speed and general messages)" )
;
parseOptionsGeneric (desc, vm, argc, argv);
checkCreateSSGrammarOptions (vm);
if ( (*vm) [HifstConstants::kPatternstoinstancesMaxspan.c_str() ].as<unsigned>()
< (*vm) [ HifstConstants::kCykparserHrmaxheight.c_str()].as<unsigned>() ) {
LERROR ( HifstConstants::kPatternstoinstancesMaxspan <<
" cannot be smaller than " << HifstConstants::kCykparserHrmaxheight);
exit (EXIT_FAILURE );
}
if ( (*vm) [HifstConstants::kFeatureweights.c_str()].as<std::string>() != ""
&& ( (*vm) [HifstConstants::kLmFeatureweights.c_str()].as<std::string>() != ""
|| (*vm) [HifstConstants::kGrammarFeatureweights.c_str()].as<std::string>() !=
"" ) ) {
LWARN ("Program option featureweights OVERRIDES grammar.featureweights and lm.featureweights!!");
}
} catch ( std::exception& e ) {
cerr << "error: " << e.what() << "\n";
exit ( EXIT_FAILURE );
} catch ( ... ) {
cerr << "Exception of unknown type!\n";
exit ( EXIT_FAILURE );
}
LINFO ( "Configuration loaded" );
};
int Z3DTexture::getBypePerPixel(GLint internalFormat)
{
int bpp = 0;
switch (internalFormat) {
case 1:
case GL_COLOR_INDEX:
case GL_RED:
case GL_GREEN:
case GL_BLUE:
case GL_ALPHA:
case GL_R8:
case GL_R8_SNORM:
case GL_INTENSITY:
case GL_LUMINANCE:
case GL_DEPTH_COMPONENT:
case GL_R3_G3_B2:
case GL_RGBA2:
case GL_R8I:
case GL_R8UI:
bpp = 1;
break;
case 2:
case GL_LUMINANCE_ALPHA:
case GL_INTENSITY16:
case GL_R16:
case GL_R16F:
case GL_R16_SNORM:
case GL_RG8:
case GL_RG8_SNORM:
case GL_DEPTH_COMPONENT16:
case GL_RGBA4:
case GL_R16I:
case GL_R16UI:
case GL_RG8I:
case GL_RG8UI:
bpp = 2;
break;
case GL_RGB:
case GL_BGR:
case GL_RGB8:
case GL_RGB8I:
case GL_RGB8UI:
case GL_SRGB8:
case GL_RGB8_SNORM:
case GL_DEPTH_COMPONENT24:
bpp = 3;
break;
case GL_RGBA:
case GL_RGBA8:
case GL_RGBA8_SNORM:
case GL_BGRA:
case GL_DEPTH_COMPONENT32:
case GL_DEPTH_COMPONENT32F:
case GL_R32F:
case GL_RG16:
case GL_RG16F:
case GL_RG16_SNORM:
case GL_SRGB8_ALPHA8:
case GL_R32I:
case GL_R32UI:
case GL_RG16I:
case GL_RG16UI:
case GL_RGBA8I:
case GL_RGBA8UI:
bpp = 4;
break;
case GL_RGB16:
case GL_RGB16I:
case GL_RGB16UI:
case GL_RGB16F:
case GL_RGB16_SNORM:
bpp = 6;
break;
case GL_RGBA16:
case GL_RGBA16F:
case GL_RGBA16I:
case GL_RGBA16UI:
case GL_RG32F:
case GL_RG32I:
case GL_RG32UI:
bpp = 8;
break;
case GL_RGB32F:
case GL_RGB32I:
case GL_RGB32UI:
bpp = 12;
break;
case GL_RGBA32I:
case GL_RGBA32UI:
case GL_RGBA32F:
bpp = 16;
break;
default:
LWARN() << "unknown internal format";
break;
}
return bpp;
}
int Z3DTexture::getBypePerPixel(GLint dataFormat, GLenum dataType)
{
int numChannels = 0;
switch (dataFormat) {
case 1:
case GL_COLOR_INDEX:
case GL_RED:
case GL_GREEN:
case GL_BLUE:
case GL_ALPHA:
case GL_INTENSITY:
case GL_LUMINANCE:
case GL_DEPTH_COMPONENT:
case GL_ALPHA_INTEGER_EXT:
numChannels = 1;
break;
case 2:
case GL_LUMINANCE_ALPHA:
numChannels = 2;
break;
case 3:
case GL_RGB:
case GL_BGR:
numChannels = 3;
break;
case 4:
case GL_RGBA:
case GL_BGRA:
case GL_RGBA16:
case GL_RGBA16F_ARB:
numChannels = 4;
break;
default:
LWARN() << "unknown data format";
}
int typeSize = 0;
switch (dataType) {
case GL_BYTE:
case GL_UNSIGNED_BYTE:
case GL_UNSIGNED_INT_8_8_8_8_REV:
case GL_UNSIGNED_INT_8_8_8_8:
typeSize = 1;
break;
case GL_SHORT:
case GL_UNSIGNED_SHORT:
typeSize = 2;
break;
case GL_INT:
case GL_UNSIGNED_INT:
case GL_FLOAT:
typeSize = 4;
break;
default:
LWARN() << "unknown data type";
}
return typeSize * numChannels;
}
Window::~Window()
{
if (_overlays.size()) {
LWARN("Window", "not all video overlays destroyed: size=%d", _overlays.size() );
}
}
BOOL parse_incoming_packet(BufferPtr *buffer, IncomingPacket *packet) {
XmlAttr attr;
Buffer stanza_name;
char erase;
int retval, erase_index = 0;
memset(packet, 0, sizeof(*packet));
packet->header.data = buffer->data;
// Parse stanza name
if (xmlfsm_node_name(buffer, &stanza_name) == XMLPARSE_FAILURE) {
LWARN("dropping: stanza name parsing failure");
return FALSE;
}
if (
!BUF_EQ_LIT("message", &stanza_name) &&
!BUF_EQ_LIT("presence", &stanza_name) &&
!BUF_EQ_LIT("iq", &stanza_name)) {
LWARN("dropping: unknown stanza name '%.*s'", stanza_name.size, stanza_name.data);
return FALSE;
}
packet->name = *stanza_name.data;
packet->header.data +=
(packet->name == 'm' ? 8 :
packet->name == 'p' ? 9 : 3);
packet->header.end = buffer->data;
// Parse attrs
while ((retval = xmlfsm_next_attr(buffer, &attr)) == XMLPARSE_SUCCESS) {
packet->header.end = buffer->data;
erase = 1;
if (BPT_EQ_LIT("from", &attr.name)) {
if (!jid_struct(&attr.value, &packet->real_from)) {
LWARN("'from' jid malformed: '%.*s'",
BPT_SIZE(&attr.value), attr.value.data);
return FALSE;
}
} else if (BPT_EQ_LIT("to", &attr.name)) {
if (!jid_struct(&attr.value, &packet->proxy_to)) {
LWARN("'to' jid malformed: '%.*s'",
BPT_SIZE(&attr.value), attr.value.data);
return FALSE;
}
} else if (BPT_EQ_LIT("type", &attr.name)) {
if (!set_type(&attr.value, packet)) {
return FALSE;
}
//erase = packet->name == 'i';
} else {
erase = 0;
}
if (erase) {
packet->erase[erase_index].data = attr.name.data;
packet->erase[erase_index].end = attr.value.end+1;
++erase_index;
}
}
if (retval == XMLPARSE_FAILURE) {
LWARN("dropping: attr parsing failure");
return FALSE;
}
if (JID_EMPTY(&packet->real_from) || JID_EMPTY(&packet->proxy_to)) {
LWARN("dropping: not routable");
return FALSE;
}
if (packet->name == 'm' || packet->name == 'p') {
if (BPT_BLANK(&packet->proxy_to.node)) {
// Messages and presences MUST contain room node name
LDEBUG("dropping: message/presence without node name");
return FALSE;
}
if (packet->name == 'm' && BPT_NULL(&packet->proxy_to.resource) && packet->type != 'g') {
LDEBUG("dropping: wrong message type for groupchat");
return FALSE;
}
}
// Set inner data
if (retval == XMLNODE_EMPTY) {
packet->inner.data = 0;
packet->inner.end = 0;
} else {
packet->inner.data = buffer->data;
packet->inner.end = buffer->end - stanza_name.size - 3;
}
return TRUE;
}
