//------------------------------------------------------------------------------------------------------------------
///this checks if 5 parameters are correct
bool pMolStackList::checkParameters(int type0, int type1, int type2, int type3, int type4, QString &error)
{
//only one parameter - no good
if (length()<5)
{ error = "Not Enough parameters";
return false;
}
if (!checkParameter(at(0),type0))
{ error = "paramater 1 - wrong type";
return false;
};
if (!checkParameter(at(1),type1))
{ error = "paramater 2 - wrong type";
return false;
};
if (!checkParameter(at(2),type2))
{ error = "paramater 3 - wrong type";
return false;
};
if (!checkParameter(at(3),type3))
{ error = "paramater 4 - wrong type";
return false;
};
if (!checkParameter(at(4),type4))
{ error = "paramater 5 - wrong type";
return false;
};
return true;
};
void initConfig() {
_config_param = parameter_new();
/* config-dir */
char *home = getenv("HOME");
if (home == NULL) {
error_set(ERROR_CONFIG_HOME, "Unable to get home directory.");
return;
}
STRINGBUFFER *tmp_cfg = stringbuffer_new();
/* config-directory */
stringbuffer_append(tmp_cfg, home);
stringbuffer_append(tmp_cfg, SEPARATOR);
stringbuffer_append(tmp_cfg, CONFIG_DIRECTORY);
_config_directory = stringbuffer_clone(tmp_cfg);
/* config-file */
stringbuffer_append(tmp_cfg, SEPARATOR);
stringbuffer_append(tmp_cfg, CONFIG_FILE);
_config_file = stringbuffer_clone(tmp_cfg);
printf ( "CONFIGFILE:\t%s\n", stringbuffer_getTextPointer(_config_file) );
stringbuffer_free(tmp_cfg);
/* load config */
_config_param = parameter_new();
parameter_loadFromFile(_config_param, stringbuffer_getTextPointer(_config_file));
checkParameter(CONFIG_DIRECTORY_STORE_KEY, CONFIG_DIRECTORY_STORE_VALUE);
checkParameter(CONFIG_DIRECTORY_RESTORE_KEY, CONFIG_DIRECTORY_RESTORE_VALUE);
checkParameter(CONFIG_DIRECTORY_KEY_KEY, CONFIG_DIRECTORY_KEY_VALUE);
/* make directories */
char *tmp_dir;
/* make store directory */
tmp_dir = parameter_get(_config_param, CONFIG_DIRECTORY_STORE_KEY);
assert(tmp_dir);
if (!directory_exists(tmp_dir))
directory_create(tmp_dir);
free(tmp_dir);
/* make restore directory */
tmp_dir = parameter_get(_config_param, CONFIG_DIRECTORY_RESTORE_KEY);
assert(tmp_dir);
if (!directory_exists(tmp_dir))
directory_create(tmp_dir);
free(tmp_dir);
/* make key directory */
tmp_dir = parameter_get(_config_param, CONFIG_DIRECTORY_KEY_KEY);
assert(tmp_dir);
if (!directory_exists(tmp_dir))
directory_create(tmp_dir);
free(tmp_dir);
}
void Preprocessing::read(int argc, char **argv, int rank)
{
parser.setCommandLine(argc, argv);
int gen_section = parser.addSection("General options");
fn_out = parser.getOption("--o", "Output rootname", "particles");
// Dont allow for directories here!
fn_out = fn_out.removeDirectories();
int particle_section = parser.addSection("Particle selection");
fns_coords_in = parser.getOption("--coord_files", "The coordinate files for all particles to be output (may contain wildcards e.g. \"mics/*.box\" or \"mics/???.star\" )","");
fn_star_in = parser.getOption("--mic_star", "The STAR file with all (selected) micrographs to extract particles from","");
fn_pick_suffix = parser.getOption("--coord_suffix", "The suffix for the coordinate files, e.g. \"_picked.star\" or \".box\"","");
int extract_section = parser.addSection("Particle extraction");
do_extract = parser.checkOption("--extract", "Extract all particles from the micrographs");
extract_size = textToInteger(parser.getOption("--extract_size", "Size of the box to extract the particles in (in pixels)", "-1"));
extract_bias_x = textToInteger(parser.getOption("--extract_bias_x", "Bias in X-direction of picked particles (this value in pixels will be added to the coords)", "0"));
extract_bias_y = textToInteger(parser.getOption("--extract_bias_y", "Bias in Y-direction of picked particles (this value in pixels will be added to the coords)", "0"));
do_movie_extract = parser.checkOption("--extract_movies", "Extract particles from movie stacks (e.g. from DDDs)");
avg_n_frames = textToInteger(parser.getOption("--avg_movie_frames", "Average over this number of individual movie frames", "1"));
movie_first_frame = textToInteger(parser.getOption("--first_movie_frame", "Extract from this movie frame onwards", "1"));
movie_first_frame--; // (start counting at 0, not 1)
movie_last_frame = textToInteger(parser.getOption("--last_movie_frame", "Extract until this movie frame (default=all movie frames)", "0"));
movie_last_frame--; // (start counting at 0, not 1)
fn_movie = parser.getOption("--movie_rootname", "Common name to relate movies to the single micrographs (e.g. mic001_movie.mrcs related to mic001.mrc)", "movie");
int perpart_section = parser.addSection("Particle operations");
do_project_3d = parser.checkOption("--project3d", "Project sub-tomograms along Z to generate 2D particles");
scale = textToInteger(parser.getOption("--scale", "Re-scale the particles to this size (in pixels)", "-1"));
window = textToInteger(parser.getOption("--window", "Re-window the particles to this size (in pixels)", "-1"));
do_normalise = parser.checkOption("--norm", "Normalise the background to average zero and stddev one");
bg_radius = textToInteger(parser.getOption("--bg_radius", "Radius of the circular mask that will be used to define the background area (in pixels)", "-1"));
white_dust_stddev = textToFloat(parser.getOption("--white_dust", "Sigma-values above which white dust will be removed (negative value means no dust removal)","-1"));
black_dust_stddev = textToFloat(parser.getOption("--black_dust", "Sigma-values above which black dust will be removed (negative value means no dust removal)","-1"));
do_invert_contrast = parser.checkOption("--invert_contrast", "Invert the contrast in the input images");
fn_operate_in = parser.getOption("--operate_on", "The operations above are applied to all extracted particles. Use this option to operate on an input stack/STAR file", "");
// Initialise verb for non-parallel execution
verb = 1;
if (!( checkParameter(argc, argv, "--o") || checkParameter(argc, argv, "--operate_on") ))
REPORT_ERROR("Provide either --o or --operate_on");
// Check for errors in the command-line option
if (parser.checkForErrors())
REPORT_ERROR("Errors encountered on the command line (see above), exiting...");
}
void CmdAdd::addMain(const Path & path) {
// For a generic approach we insert the path
arguments->insert(arguments->begin() + 1, path.toString());
std::string soort = getArgument(2);
checkParameter(soort);
insert(path, soort);
}
int
OptionsParser::check(const char* arg1, const char* arg2, bool& ok) {
// the first argument should be an option
// (only the second may be a free string)
if (!checkParameter(arg1)) {
ok = false;
return 1;
}
OptionsCont& oc = OptionsCont::getOptions();
// process not abbreviated switches
if (!isAbbreviation(arg1)) {
std::string tmp(arg1 + 2);
size_t idx1 = tmp.find('=');
// check whether a parameter was submitted
if (idx1 != std::string::npos) {
ok &= oc.set(tmp.substr(0, idx1), tmp.substr(idx1 + 1));
} else {
if (arg2 == 0 || (oc.isBool(convert(arg1 + 2)) && arg2[0] == '-')) {
ok &= oc.set(convert(arg1 + 2), "true");
} else {
ok &= oc.set(convert(arg1 + 2), convert(arg2));
return 2;
}
}
return 1;
}
// go through the abbreviated switches
for (int i = 1; arg1[i] != 0; i++) {
// set boolean switches
if (oc.isBool(convert(arg1[i]))) {
if (arg2 == 0 || arg2[0] == '-' || arg1[i + 1] != 0) {
ok &= oc.set(convert(arg1[i]), "true");
} else {
ok &= oc.set(convert(arg1[i]), convert(arg2));
return 2;
}
// set non-boolean switches
} else {
// check whether the parameter comes directly after the switch
// and process if so
if (arg2 == 0 || arg1[i + 1] != 0) {
ok &= processNonBooleanSingleSwitch(oc, arg1 + i);
return 1;
// process parameter following after a space
} else {
ok &= oc.set(convert(arg1[i]), convert(arg2));
// option name and attribute were in two arguments
return 2;
}
}
}
// all switches within the current argument were boolean switches
return 1;
}
//------------------------------------------------------------------------------------------------------------------
///this checks if 2 parameters are correct
bool pMolStackList::checkParameters(int type0, int type1, QString &error)
{
//only one parameter - no good
if (length()<2)
{ error = "wrong number of parameters";
return false;
}
if (!checkParameter(at(0),type0))
{ error = "paramater 1 - wrong type";
return false;
};
if (!checkParameter(at(1),type1))
{ error = "paramater 2 - wrong type";
return false;
};
return true;
};
int argFunctions::checkDir() {
if (checkParameter("-d")) {
const char *directory = getValue("-d");
int correctDirectory = chdir(directory);
if (correctDirectory != 0) {
if (verboseMode) {
std::cout << "Invalid directory. Use -h to see help/correct usage.\nTerminating...\n";
}
return -1; //Error code.
}
}
return 0;
}
OsmAnd::RoutingRulesetContext::RoutingRulesetContext(RoutingProfileContext* owner_, const std::shared_ptr<RoutingRuleset>& ruleset_, QHash<QString, QString>* contextValues_)
: owner(owner_)
,_ruleset(new RoutingRuleset(ruleset_->owner, ruleset_->type))
, ruleset(_ruleset)
, contextValues(_contextValues)
{
if(contextValues_)
_contextValues = *contextValues_;
for(std::shared_ptr<RoutingRuleExpression> rt : ruleset_->_expressions){
if(checkParameter(rt, _contextValues)){
_ruleset->_expressions.push_back(rt);
}
}
}
void _SetDataFilterParameters (_String const& name, _DataSetFilter const& filter) {
setParameter (WrapInNamespace ("species", &name), filter.NumberSpecies());
setParameter (WrapInNamespace ("sites", &name), filter.GetSiteCountInUnits());
_Parameter size_cutoff;
checkParameter (defaultLargeFileCutoff,size_cutoff, 100000.);
if (filter.GetSiteCount() < size_cutoff) {
setParameter(WrapInNamespace("site_map", &name), new _Matrix (filter.theOriginalOrder), nil, false);
setParameter(WrapInNamespace("site_freqs", &name), new _Matrix (filter.theFrequencies), nil, false);
}
if (filter.NumberSpecies() < size_cutoff) {
setParameter(WrapInNamespace("sequence_map", &name), new _Matrix (filter.theNodeMap), nil, false);
}
}
long MessageHandler::checkMessageParameters(long messageid, const Message &msg)
{
int i;
int parameterid = -1;
for(i = 0; i < msg.numParameters(); i++)
{
const MessageParameter &mp = msg.getParameter(i);
parameterid = checkParameter(messageid, mp);
if(parameterid < 0) parameterid = insertParameter(messageid, mp);
}
if(msg.numParameters() == 0) return 0;
return parameterid;
}
void ProcessEvent(unsigned int me, time_type now, int event_type, event_content_type *event_content, unsigned int size, void *pointer) {
state_type *state = (state_type *) pointer;
char *conf_file = NULL, *conf_file_1 = NULL, *conf_file_2 = NULL;
simulation_conf configuration;
int i;
// Upon reception of INIT, install the simulation state
if (event_type == INIT) {
// Get the file to load the configuration from
for (i = 0; i < size; i += 2) {
if (checkParameter("conf-file", event_content, i)) {
conf_file = getParameter(event_content, i+1);
}
else if(checkParameter("conf-file_1", event_content, i)) {
conf_file_1 = getParameter(event_content, i+1);
}
else if(checkParameter("conf-file_2", event_content, i)) {
conf_file_2 = getParameter(event_content, i+1);
}
}
// Sanity check
if (conf_file == NULL) {
fprintf(stderr, "Configuration file not specified. Please set conf-file before running the simulation.\n");
exit(-1);
}
// Try to load the configuration
if (!configuration_parse(&configuration, conf_file)) {
fprintf(stderr, "Error loading configuration file %s\n", conf_file);
exit(-1);
}
if(configuration.client_conf.data_items_access_distribution==TOPKEYS){
if(!configuration_parse_top_keys(&configuration,"PUTS",conf_file_1)){
fprintf(stderr, "Error loading puts topkeys' configuration file");
exit(-1);
}
if(!configuration_parse_top_keys(&configuration,"GETS",conf_file_2)){
fprintf(stderr, "Error loading gets topkeys' configuration file");
exit(-1);
}
/*int x;
for(x=0; x<configuration.cache_objects;x++){
if(configuration.client_conf.topkeys_cache_objects_get_probability[x]==0) break;
printf("%f\n",configuration.client_conf.topkeys_cache_objects_get_probability[x]);
}*/
}
// Allocate simulation state
state = (state_type *) malloc(sizeof(state_type));
SetState(state);
// Store the global configuration
state->start_stat_time = configuration.start_stat_time;
state->num_clients = configuration.num_clients;
state->num_servers = configuration.num_servers;
state->cache_objects = configuration.cache_objects;
state->object_replication_degree = configuration.object_replication_degree;
state->average_server_to_server_net_delay = configuration.average_server_to_server_net_delay;
state->average_client_to_server_net_delay = configuration.average_client_to_server_net_delay;
// Now, depending on who I am, store the remainder of the configuration
switch (who_am_i(me, state)) {
case CLIENT:
state->type.client_state.configuration = configuration.client_conf;
break;
case SERVER:
state->type.server_state.configuration = configuration.server_conf;
break;
default:
fprintf(stderr, "Internal error at %s:%d\n", __FILE__, __LINE__);
break;
}
}
// Dispatch the event to the correct handler
switch (who_am_i(me, state)) {
case CLIENT:
CLIENT_ProcessEvent(me, now, event_type, event_content, size, state);
break;
case SERVER:
SERVER_ProcessEvent(me, now, event_type, event_content, size, state);
break;
default:
fprintf(stderr, "Internal error at %s:%d\n", __FILE__, __LINE__);
break;
}
}
void CmdAdd::addToGroup(const Path & path) {
/*
* TODO vrij veel code duplicatie met cmdInsert.
*
* Alles is hetzelfde buiten dat we methode "add"
* van specification gebruiken ipv "insert"
*/
std::string soort = getArgument(2);
checkParameter(soort);
if (soort == "text") {
std::string vraag = getRemainingArgument(3);
bool isOptional = checkForOptional(vraag);
QuestText* question = new QuestText(path, vraag, isOptional);
specification->add(path, question);
} else if (soort == "choice") {
std::vector<std::string>* questionChoices =
QuestChoice::GetQuestionChoices();
if (questionChoices) {
std::string vraag = getRemainingArgument(3);
bool isOptional = checkForOptional(vraag);
QuestChoice* question = new QuestChoice(path, vraag, isOptional,
questionChoices);
specification->add(path, question);
std::cout << "Vraag (" << vraag << ") toegevoegd aan groep " << path
<< "." << std::endl;
} else {
std::cout << "Niet genoeg geldige antwoorden." << std::endl;
}
} else if (soort == "bool") {
std::string vraag = getRemainingArgument(3);
bool isOptional = checkForOptional(vraag);
QuestBool* question = new QuestBool(path, vraag, isOptional);
specification->add(path, question);
std::cout << "Vraag (" << vraag << ") toegevoegd aan groep " << path
<< "." << std::endl;
} else if (soort == "scale") {
std::string parLow = getArgument(3);
std::string parHigh = getArgument(4);
int low;
int high;
try {
low = std::stoi(parLow);
high = std::stoi(parHigh);
} catch (const std::invalid_argument & ex) {
throw ScaleParametersNotValid(parLow, parHigh);
}
if (low >= high) {
throw ScaleParametersNotValid(parLow, parHigh);
}
std::string vraag = getRemainingArgument(5);
bool isOptional = checkForOptional(vraag);
QuestScale* question = new QuestScale(path, vraag, isOptional, low, high);
specification->add(path, question);
std::cout << "Vraag (" << vraag << ") toegevoegd aan groep " << path
<< "." << std::endl;
}
}
//__________________________________________________________________________________
void mpiNormalLoop (int rank, int size, _String & baseDir)
{
long senderID = 0;
ReportWarning ("Entered mpiNormalLoop");
_String* theMessage = MPIRecvString (-1,senderID), // listen for messages from any node
* resStr = nil,
_bgmSwitch ("_BGM_SWITCH_"),
css("_CONTEXT_SWITCH_MPIPARTITIONS_");
while (theMessage->sLength)
{
setParameter (mpiNodeID, (_Parameter)rank);
setParameter (mpiNodeCount, (_Parameter)size);
//ReportWarning (*theMessage);
DeleteObject (resStr);
resStr = nil;
if (theMessage->Equal (&css) )
{
mpiPartitionOptimizer = true;
ReportWarning ("Switched to mpiOptimizer loop");
MPISendString(css,senderID);
mpiOptimizerLoop (rank,size);
ReportWarning ("Returned from mpiOptimizer loop");
mpiPartitionOptimizer = false;
pathNames && & baseDir;
}
else if ( theMessage->Equal (&_bgmSwitch) )
{
ReportWarning ("Received signal to switch to mpiBgmLoop");
MPISendString (_bgmSwitch, senderID); // feedback to source to confirm receipt of message
mpiBgmLoop (rank, size);
ReportWarning ("Returned from mpiBgmLoop");
}
else
{
if (theMessage->beginswith ("#NEXUS"))
{
_String msgCopy (*theMessage);
ReportWarning ("Received a function to optimize");
ReadDataSetFile (nil,true,theMessage);
ReportWarning ("Done with the optimization");
_Variable* lfName = FetchVar(LocateVarByName(MPI_NEXUS_FILE_RETURN));
if (lfName)
{
resStr = (_String*)(lfName->Compute()->toStr());
}
else
{
long f = LocateVarByName (lf2SendBack);
if (f>=0)
lfName = FetchVar(f);
if (!(lfName&&(lfName->ObjectClass()==STRING)))
{
_String errMsg ("Malformed MPI likelihood function optimization request - missing LF name to return.\n\n\n");
errMsg = errMsg & msgCopy;
FlagError (errMsg);
break;
}
f = likeFuncNamesList.Find (((_FString*)lfName->Compute())->theString);
if (f<0)
{
_String errMsg ("Malformed MPI likelihood function optimization request - invalid LF name to return.\n\n\n");
errMsg = errMsg & msgCopy;
FlagError (errMsg);
break;
}
_Parameter pv;
checkParameter (shortMPIReturn, pv ,0);
resStr = new _String (1024L,true);
checkPointer (resStr);
((_LikelihoodFunction*)likeFuncList (f))->SerializeLF(*resStr,pv>0.5?5:2);
resStr->Finalize();
}
}
else
{
_ExecutionList exL (*theMessage);
/*printf ("Received:\n %s\n", ((_String*)exL.toStr())->sData);*/
_PMathObj res = exL.Execute();
resStr = res?(_String*)res->toStr():new _String ("0");
}
checkPointer (resStr);
DeleteObject (theMessage);
MPISendString(*resStr,senderID);
_Parameter keepState = 0.0;
checkParameter (preserveSlaveNodeState, keepState, 0.0);
if (keepState < 0.5)
{
PurgeAll (true);
pathNames && & baseDir;
}
}
theMessage = MPIRecvString (-1,senderID);
}
/*MPISendString(empty,senderID);*/
DeleteObject (resStr);
DeleteObject (theMessage);
}
void TrainModelNN (_String* model, _String* matrix)
{
_String errMsg;
long modelIdx = modelNames.Find(model);
_Parameter verbI;
checkParameter (VerbosityLevelString, verbI, 0.0);
char buffer [128];
if (modelIdx < 0) {
errMsg = *model & " did not refer to an existring model";
} else {
_Variable* boundsMatrix = FetchVar (LocateVarByName (*matrix));
if (boundsMatrix && (boundsMatrix->ObjectClass() == MATRIX)) {
_Matrix * bmatrix = (_Matrix*) boundsMatrix->GetValue ();
if (bmatrix->IsAStringMatrix() && (bmatrix->GetVDim () == 3)) {
_Variable* modelMatrix = LocateVar (modelMatrixIndices.lData[modelIdx]);
_SimpleList modelVariableList;
{
_AVLList mvla (&modelVariableList);
modelMatrix->ScanForVariables (mvla, true);
mvla.ReorderList();
}
if (bmatrix->GetHDim () == modelVariableList.lLength) {
// now map model variables to bounds matrix
_SimpleList variableMap;
_String *myName;
for (long k = 0; k < modelVariableList.lLength; k++) {
myName = ((_FString*)bmatrix->GetFormula(k,0)->Compute())->theString;
long vID = LocateVarByName (*myName);
if (vID < 0) {
break;
}
vID = variableNames.GetXtra (vID);
vID = modelVariableList.Find(vID);
if (vID < 0) {
break;
}
variableMap << vID;
}
if (variableMap.lLength == modelVariableList.lLength) {
_Matrix vBounds (variableMap.lLength,2, false, true);
long k2 = 0;
for (; k2 < variableMap.lLength; k2++) {
_Parameter lb = ((_FString*)bmatrix->GetFormula(k2,1)->Compute())->theString->toNum(),
ub = ((_FString*)bmatrix->GetFormula(k2,2)->Compute())->theString->toNum();
if ( ub>lb || k2) {
vBounds.Store (k2,0,lb);
vBounds.Store (k2,1,ub);
if (ub<=lb && vBounds (k2-1,0) <= vBounds (k2-1,1) && (!CheckEqual(vBounds (k2-1,0),0.0) || !CheckEqual(vBounds (k2-1,1),1.0))) {
break;
}
}
}
if (k2 == modelVariableList.lLength) {
// set up the sampling now
_String fName = ProcessLiteralArgument (&ModelNNFile,nil);
FILE* nnFile = doFileOpen (fName.getStr(), "w");
if (nnFile) {
_Matrix* modelMatrix = (_Matrix*) LocateVar(modelMatrixIndices.lData[modelIdx])->GetValue();
_Parameter mainSteps,
checkSteps,
errorTerm,
loopMax,
hiddenNodes,
absError,
nn1,
nn2;
long fullDimension = modelMatrix->GetHDim() * modelMatrix->GetVDim();
checkParameter (ModelNNTrainingSteps, mainSteps, 10000.0);
checkParameter (ModelNNVerificationSample, checkSteps, 500.0);
checkParameter (ModelNNPrecision, errorTerm, 0.01);
checkParameter (ModelNNTrainingSteps, loopMax, 10);
checkParameter (ModelNNHiddenNodes, hiddenNodes, 5);
checkParameter (ModelNNLearningRate, nn1, .3);
checkParameter (ModelNNPersistenceRate, nn2, .1);
Net** matrixNet = new Net* [fullDimension] ;
for (long i = 0; i < fullDimension; i++) {
checkPointer (matrixNet [i] = new Net (variableMap.lLength,(long)hiddenNodes,1,errorTerm,nn1,nn2,100,200,true));
//matrixNet[i]->verbose = true;
}
checkPointer (matrixNet);
_List tIn,
tOut;
FILE* varSamples = doFileOpen ("variableSamples.out", "w");
fprintf (varSamples, "%s" ,LocateVar(modelVariableList.lData[0])->GetName()->getStr());
for (long vc = 1; vc < modelVariableList.lLength; vc++) {
fprintf (varSamples, ",%s" ,LocateVar(modelVariableList.lData[variableMap.lData[vc]])->GetName()->getStr());
}
fprintf (varSamples, "\n");
for (long itCount = 0; itCount < loopMax; itCount ++) {
if (verbI > 5) {
snprintf (buffer, sizeof(buffer), "\nNeural Network Pass %ld. Building a training set...\n", itCount);
BufferToConsole (buffer);
}
while (tIn.countitems() < mainSteps) {
NNMatrixSampler (0, vBounds, modelVariableList, variableMap, modelMatrix, tIn, tOut);
}
_Matrix inData (mainSteps, variableMap.lLength, false, true);
_Parameter *md = inData.theData;
for (long matrixC = 0; matrixC < mainSteps; matrixC++) {
_Parameter * ed = ((_Matrix*)tIn (matrixC))->theData;
fprintf (varSamples, "\n%g",*ed);
*md = *ed;
ed++;
md++;
for (long entryC = 1; entryC < variableMap.lLength; entryC++, ed++, md++) {
*md = *ed;
fprintf (varSamples, ",%g", *md);
}
}
tIn.Clear();
if (verbI > 5) {
BufferToConsole ( "Done Building Training Set. Training...\n");
}
long lastDone = 0;
for (long cellCount = 0; cellCount < fullDimension; cellCount++) {
Net* thisCell = matrixNet[cellCount];
_Matrix outVector (mainSteps, 1, false, true);
for (long oc = 0; oc < mainSteps; oc++) {
outVector.theData[oc] = ((_Matrix*)tOut(oc))->theData[cellCount];
}
thisCell->studyAll (inData.theData, outVector.theData, mainSteps);
long nowDone = (cellCount+1)*100./fullDimension;
if (nowDone > lastDone) {
snprintf (buffer, sizeof(buffer),"%ld%% done\n", lastDone = nowDone);
BufferToConsole (buffer);
}
}
tOut.Clear();
if (verbI > 5) {
BufferToConsole ( "Done Training. Resampling...\n");
}
_PMathObj tObj = _Constant(0).Time();
_Parameter time1 = tObj->Value(),
time2;
while (tIn.countitems() < checkSteps) {
NNMatrixSampler (0, vBounds, modelVariableList, variableMap, modelMatrix, tIn, tOut);
}
absError = 0.0;
DeleteObject (tObj);
tObj = _Constant(0).Time();
time2 = tObj->Value();
if (verbI > 5) {
snprintf (buffer, sizeof(buffer),"Done Resampling in %g seconds. Computing Error...\n", time2-time1);
BufferToConsole (buffer);
}
_Parameter maxValT,
maxValE;
for (long verCount = 0; verCount < checkSteps; verCount++) {
_Parameter* inData = ((_Matrix*)tIn(verCount))->theData,
* outData = ((_Matrix*)tOut(verCount))->theData;
for (long cellCount = 0; cellCount < fullDimension; cellCount++) {
Net *thisCell = matrixNet[cellCount];
_Parameter estVal = thisCell->eval(inData)[0],
trueVal = outData[cellCount],
localError;
localError = estVal-trueVal;
if (localError < 0) {
localError = -localError;
}
if (absError < localError) {
maxValT = trueVal;
maxValE = estVal;
absError = localError;
}
}
}
DeleteObject (tObj);
tObj = _Constant(0).Time();
time1 = tObj->Value();
DeleteObject (tObj);
if (verbI > 5) {
snprintf (buffer, sizeof(buffer), "Done Error Checking in %g seconds. Got max abs error %g on the pair %g %g\n", time1-time2, absError, maxValT, maxValE);
BufferToConsole (buffer);
}
if (absError <= errorTerm) {
break;
}
}
if (absError > errorTerm) {
ReportWarning (_String("Couldn't achive desired precision in TrainModelNN. Achieved error of ") & absError);
}
fclose (varSamples);
fprintf (nnFile,"{{\n\"%s\"", LocateVar(modelVariableList.lData[0])->GetName()->getStr());
_Matrix newBounds (modelVariableList.lLength, 2, false, true);
if (vBounds(0,0)>vBounds(0,1)) {
newBounds.Store (variableMap.lData[0],0,0.);
newBounds.Store (variableMap.lData[0],1,1.);
} else {
newBounds.Store (variableMap.lData[0],0,vBounds(0,0));
newBounds.Store (variableMap.lData[0],1,vBounds(0,1));
}
for (long varCounter = 1; varCounter < modelVariableList.lLength; varCounter ++) {
fprintf (nnFile,",\n\"%s\"", LocateVar(modelVariableList.lData[varCounter])->GetName()->getStr());
if (vBounds(varCounter,0)>vBounds(varCounter,1)) {
newBounds.Store (variableMap.lData[varCounter],0,0.);
newBounds.Store (variableMap.lData[varCounter],1,1.);
} else {
newBounds.Store (variableMap.lData[varCounter],0,vBounds(varCounter,0));
newBounds.Store (variableMap.lData[varCounter],1,vBounds(varCounter,1));
}
}
fprintf (nnFile,"\n}}\n");
newBounds.toFileStr (nnFile);
for (long i2 = 0; i2 < fullDimension; i2++) {
matrixNet[i2]->save(nnFile);
delete matrixNet [i2];
}
fclose (nnFile);
delete matrixNet;
} else {
errMsg = _String ("Failed to open ") & fName & " for writing";
}
} else {
errMsg = _String ("Invalid variable bounds in row ") & (k2+1) & " of the bounds matrix";
}
} else {
errMsg = *myName & " was not one of the model parameters";
}
} else {
errMsg = *matrix & " must be a have the same number of rows as the number of model parameters";
}
} else {
errMsg = *matrix & " must be a string matrix with 3 columns";
}
} else {
errMsg = *matrix & " was not the identifier of a valid matrix variable";
}
}
if (errMsg.sLength) {
errMsg = errMsg & _String(" in call to TrainModelNN.");
WarnError (errMsg);
}
}
void _HYConsoleWindow::_PaintStatusBar(Ptr,bool force)
{
if (GTK_WIDGET_MAPPED (theWindow)) {
_Parameter vL;
checkParameter (VerbosityLevelString, vL, 0.0);
if (vL<-0.5 && !force) {
clock_t curMeasure = clock();
_Parameter diff = 1.0/CLOCKS_PER_SEC*(curMeasure-lastMeasure);
if (diff < -vL) {
return;
}
lastMeasure = curMeasure;
}
if (!stripedFillGC) {
SetUpStatusBarStuff (theWindow);
}
GdkRectangle wRC = {0,0,theWindow->allocation.width,HY_SCROLLER_WIDTH},
w2RC;
if (!_hyConsoleWindowGC) {
_hyConsoleWindowGC = gdk_gc_new (theWindow->window);
gdk_gc_set_tile (_hyConsoleWindowGC, stripedFill);
gdk_gc_set_line_attributes(_hyConsoleWindowGC,1,GDK_LINE_SOLID,GDK_CAP_NOT_LAST,GDK_JOIN_MITER);
_hyConsoleLayout = pango_layout_new (screenPContext);
pango_layout_set_width (_hyConsoleLayout, -1);
statusBarBold = pango_font_description_new ();
statusBarNormal = pango_font_description_new ();
_HYFont consoleFont = {_HY_MONO_FONT,9,HY_FONT_PLAIN};
HYFont2PangoFontDesc (consoleFont,statusBarNormal);
consoleFont.style = HY_FONT_BOLD;
HYFont2PangoFontDesc (consoleFont,statusBarBold);
}
GdkPixmap * offBitmap = gdk_pixmap_new (theWindow->window, wRC.width, wRC.height,-1);
if (offBitmap) {
gdk_gc_set_fill (_hyConsoleWindowGC,GDK_TILED);
gdk_draw_rectangle (offBitmap,_hyConsoleWindowGC,true,-1,-1,wRC.width+2,wRC.height+2);
gdk_gc_set_fill (_hyConsoleWindowGC,GDK_SOLID);
gdk_gc_set_foreground (_hyConsoleWindowGC,&_BLACKBRUSH_);
gdk_draw_line(offBitmap,_hyConsoleWindowGC,0,0,wRC.width,0);
pango_layout_set_font_description(_hyConsoleLayout,statusBarNormal);
pango_layout_set_text(_hyConsoleLayout,fileName.getStr(),fileName.sLength);
gdk_draw_layout(offBitmap,_hyConsoleWindowGC,33,wRC.height-13,_hyConsoleLayout);
if (inputStatus == 1) {
pango_layout_set_text(_hyConsoleLayout,cInput.getStr(),cInput.sLength);
} else {
pango_layout_set_text(_hyConsoleLayout,action.getStr(),action.sLength);
}
gdk_draw_layout(offBitmap,_hyConsoleWindowGC,193,wRC.height-13,_hyConsoleLayout);
gdk_gc_set_foreground (_hyConsoleWindowGC,&_DARKGREYBRUSH_);
gdk_draw_rectangle (offBitmap,_hyConsoleWindowGC,true,0,1,30,wRC.height-1);
gdk_draw_rectangle (offBitmap,_hyConsoleWindowGC,true,150,1,40,wRC.height-1);
gdk_draw_rectangle (offBitmap,_hyConsoleWindowGC,true,wRC.width-55,1,55,wRC.height-1);
gdk_gc_set_foreground (_hyConsoleWindowGC,&_WHITEBRUSH_);
pango_layout_set_font_description(_hyConsoleLayout,statusBarBold);
pango_layout_set_text(_hyConsoleLayout,cState.getStr(),cState.sLength);
gdk_draw_layout(offBitmap,_hyConsoleWindowGC,3,wRC.height-13,_hyConsoleLayout);
pango_layout_set_text(_hyConsoleLayout,cTask.getStr(),cTask.sLength);
gdk_draw_layout(offBitmap,_hyConsoleWindowGC,151,wRC.height-13,_hyConsoleLayout);
pango_layout_set_font_description(_hyConsoleLayout,statusBarNormal);
pango_layout_set_text(_hyConsoleLayout,timer.getStr(),timer.sLength);
gdk_draw_layout(offBitmap,_hyConsoleWindowGC,wRC.width-53,wRC.height-13,_hyConsoleLayout);
if (percentDone>0 || percentDone == -HY_SL_DONE) {
GdkColor blackBrush = HYColorToGDKColor((_HYColor) {
80,80,80
}),
orangeBrush = HYColorToGDKColor((_HYColor) {
255,153,102
});
gdk_gc_set_foreground (_hyConsoleWindowGC,&orangeBrush);
gdk_draw_rectangle (offBitmap,_hyConsoleWindowGC,true,wRC.width-135,wRC.height-14,(percentDone>=0?percentDone:100.)*0.75,12);
gdk_gc_set_foreground (_hyConsoleWindowGC,&blackBrush);
gdk_draw_rectangle (offBitmap,_hyConsoleWindowGC,false,wRC.width-135,wRC.height-14,75,12);
//gdk_gc_set_foreground (_hyConsoleWindowGC,&_WHITEBRUSH_);
_String pLine;
if (percentDone>=0) {
pLine = _String(percentDone)&"%";
} else {
pLine = "DONE";
}
pango_layout_set_text(_hyConsoleLayout,pLine.getStr(),pLine.sLength);
gdk_draw_layout(offBitmap,_hyConsoleWindowGC,wRC.width-107,wRC.height-13,_hyConsoleLayout);
}
}
gdk_draw_drawable (theWindow->window, _hyConsoleWindowGC, offBitmap, 0, 0, theWindow->allocation.x, theWindow->allocation.y+theWindow->allocation.height-wRC.height, -1, -1);
g_object_unref (offBitmap);
}
}
bool Get_a_URL (_String& urls, _String* fileName)
{
#ifdef __HYPHYCURL__
CURL *curl;
CURLcode res ;
curl = curl_easy_init ();
FILE * f = nil;
_String* s = nil;
char cErr [CURL_ERROR_SIZE+1];
if(curl) {
if (fileName) {
f = fopen (fileName->sData,"wb");
if (!f) {
urls = _String ("Failed to open ") & *fileName & " for writing";
return false;
}
} else {
s = new _String (8192, true);
checkPointer (s);
}
curl_easy_setopt (curl, CURLOPT_URL, urls.sData );
curl_easy_setopt (curl, CURLOPT_ERRORBUFFER, cErr);
//Do not check peer certificate, since we only ever get urls
curl_easy_setopt (curl, CURLOPT_SSL_VERIFYPEER, 0);
curl_easy_setopt (curl, CURLOPT_SSL_VERIFYHOST, 0);
if (f) {
curl_easy_setopt (curl, CURLOPT_FILE, (void*)f);
} else {
curl_easy_setopt (curl, CURLOPT_FILE, (void*)s);
}
_String ver (GetVersionString());
curl_easy_setopt (curl, CURLOPT_USERAGENT, ver.sData);
//curl_easy_setopt (curl, CURLOPT_VERBOSE, 1);
curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, (void*)(f?url2File:url2String));
_Parameter vbl = 0.0;
checkParameter (VerbosityLevelString,vbl,0.0);
if (vbl<0.5) {
curl_easy_setopt (curl,CURLOPT_NOPROGRESS,1);
}
res = curl_easy_perform (curl);
curl_easy_cleanup (curl);
if (f) {
fclose (f);
} else {
s->Finalize();
urls = *s;
DeleteObject (s);
}
if (!res) {
return true;
}
} else {
urls = "Failed to initialize CURL object";
return false;
}
urls = _String ("CURL error:") & (long)res & "." & cErr;
return false;
#else
urls = "This feature requires libcurl";
return false;
#endif
}
//__________________________________________________________________________________
void mpiNormalLoop (int rank, int size, _String & baseDir)
{
long senderID = 0;
ReportWarning ("[MPI] Entered mpiNormalLoop");
_String* theMessage = MPIRecvString (-1,senderID), // listen for messages from any node
* resStr = nil;
while (theMessage->sLength) {
setParameter (mpiNodeID, (_Parameter)rank);
setParameter (mpiNodeCount, (_Parameter)size);
//ReportWarning (*theMessage);
DeleteObject (resStr);
resStr = nil;
if (theMessage->startswith (mpiLoopSwitchToOptimize) ) {
hyphyMPIOptimizerMode = theMessage->Cut(mpiLoopSwitchToOptimize.sLength,-1).toNum();
ReportWarning (_String("[MPI] Switched to mpiOptimizer loop with mode ") & hyphyMPIOptimizerMode);
MPISendString (mpiLoopSwitchToOptimize,senderID);
mpiOptimizerLoop (rank,size);
ReportWarning ("[MPI] Returned from mpiOptimizer loop");
hyphyMPIOptimizerMode = _hyphyLFMPIModeNone;
pathNames && & baseDir;
} else if ( theMessage->Equal (&mpiLoopSwitchToBGM) ) {
ReportWarning ("[MPI] Received signal to switch to mpiBgmLoop");
MPISendString (mpiLoopSwitchToBGM, senderID); // feedback to source to confirm receipt of message
mpiBgmLoop (rank, size);
ReportWarning ("[MPI] Returned from mpiBgmLoop");
} else {
if (theMessage->beginswith ("#NEXUS")) {
_String msgCopy (*theMessage);
ReportWarning ("[MPI] Received a function to optimize");
ReadDataSetFile (nil,true,theMessage);
ReportWarning ("[MPI] Done with the optimization");
_Variable* lfName = FetchVar(LocateVarByName(MPI_NEXUS_FILE_RETURN));
if (lfName) {
resStr = (_String*)(lfName->Compute()->toStr());
} else {
_FString *lfID = (_FString*)FetchObjectFromVariableByType (&lf2SendBack, STRING);
if (!lfID) {
FlagError (_String("[MPI] Malformed MPI likelihood function optimization request - did not specify the LF name to return in variable ") & lf2SendBack & ".\n\n\n" );
break;
}
long f = likeFuncNamesList.Find (lfID->theString);
if (f<0) {
FlagError ("[MPI] Malformed MPI likelihood function optimization request - LF name to return did not refer to a well-defined likelihood function.\n\n\n");
break;
}
_Parameter pv;
checkParameter (shortMPIReturn, pv ,0);
resStr = (_String*)checkPointer(new _String (1024L,true));
((_LikelihoodFunction*)likeFuncList (f))->SerializeLF(*resStr,pv>0.5?_hyphyLFSerializeModeShortMPI:_hyphyLFSerializeModeLongMPI);
resStr->Finalize();
}
} else {
_ExecutionList exL (*theMessage);
_PMathObj res = exL.Execute();
resStr = res?(_String*)res->toStr():new _String ("0");
}
checkPointer (resStr);
MPISendString(*resStr,senderID);
_Parameter keepState = 0.0;
checkParameter (preserveSlaveNodeState, keepState, 0.0);
if (keepState < 0.5) {
PurgeAll (true);
pathNames && & baseDir;
}
}
DeleteObject (theMessage);
theMessage = MPIRecvString (-1,senderID);
}
/*MPISendString(empty,senderID);*/
DeleteObject (resStr);
DeleteObject (theMessage);
}
/*
* Entry point for pg_keeper.
*/
void
KeeperMain(Datum main_arg)
{
int ret;
/* Sanity check */
checkParameter();
/* Initial setting */
KeeperMaster = keeper_node1_conninfo;
KeeperStandby = keeper_node2_conninfo;
/* Determine keeper mode of itself */
current_mode = RecoveryInProgress() ? KEEPER_STANDBY_MODE : KEEPER_MASTER_MODE;
/* Establish signal handlers before unblocking signals */
pqsignal(SIGHUP, pg_keeper_sighup);
pqsignal(SIGTERM, pg_keeper_sigterm);
/* We're now ready to receive signals */
BackgroundWorkerUnblockSignals();
/* Connect to our database */
BackgroundWorkerInitializeConnection("postgres", NULL);
exec:
if (current_mode == KEEPER_MASTER_MODE)
{
/* Routine for master_mode */
setupKeeperMaster();
ret = KeeperMainMaster();
}
else if (current_mode == KEEPER_STANDBY_MODE)
{
/* Routine for standby_mode */
setupKeeperStandby();
ret = KeeperMainStandby();
/*
* After promoting is sucessfully done, attempt to re-execute
* main routine as master mode in order to avoid to restart
* for invoking pg_keeper process again.
*/
if (ret)
{
/* Change mode to master mode */
current_mode = KEEPER_MASTER_MODE;
/* Switch master and standby connection information */
swtichMasterAndStandby();
ereport(LOG,
(errmsg("swtiched master and standby informations"),
errdetail("\"%s\" is regarded as master server, \"%s\" is regarded as standby server",
KeeperMaster, KeeperStandby)));
goto exec;
}
}
else
ereport(ERROR, (errmsg("invalid keeper mode : \"%d\"", current_mode)));
proc_exit(ret);
}
int main() {
checkParameter(std::numeric_limits<int>::min(), 0u, std::numeric_limits<unsigned int>::max());
checkParameter(std::numeric_limits<long long>::min(), 0u, std::numeric_limits<unsigned long long>::max());
checkParameter(20u, 10.1, 30.5f);
}
