/** Calculate Pearson product-moment correlation between DataSets.
* \D1 DataSet to caclculate correlation for.
* \D2 DataSet to caclulate correlation to.
* \return Pearson product-moment correlation coefficient.
*/
double DS_Math::CorrCoeff( DataSet& D1, DataSet& D2 ) {
// Check if D1 and D2 are valid types
if ( !GoodCalcType(D1) ) return 0;
if ( !GoodCalcType(D2) ) return 0;
// Check that D1 and D2 have same # data points.
int Nelements = D1.Size();
if (Nelements != D2.Size()) {
mprinterr("Error: Corr: # elements in dataset %s (%i) not equal to\n",
D1.Legend().c_str(), Nelements);
mprinterr("Error: # elements in dataset %s (%i)\n",
D2.Legend().c_str(), D2.Size());
return 0;
}
// Calculate averages
double avg1 = Avg(D1);
double avg2 = Avg(D2);
// Calculate average deviations.
double sumdiff1_2 = 0.0;
double sumdiff2_2 = 0.0;
double corr_coeff = 0.0;
//mprinterr("DATASETS %s and %s\n", c_str(), D2.c_str());
for (int i = 0; i < Nelements; i++) {
double diff1 = D1.Dval(i) - avg1;
double diff2 = D2.Dval(i) - avg2;
sumdiff1_2 += (diff1 * diff1);
sumdiff2_2 += (diff2 * diff2);
corr_coeff += (diff1 * diff2);
}
if (sumdiff1_2 == 0.0 || sumdiff2_2 == 0.0) {
mprintf("Warning: Corr: %s to %s, Normalization is 0\n",
D1.Legend().c_str(), D2.Legend().c_str());
return 0;
}
// Correlation coefficient
corr_coeff /= ( sqrt( sumdiff1_2 ) * sqrt( sumdiff2_2 ) );
//mprintf(" CORRELATION COEFFICIENT %6s to %6s IS %10.4f\n",
// D1_->c_str(), D2_->c_str(), corr_coeff );
return corr_coeff;
}
size_t nnet_predict(NNet& nnet, DataSet& data) {
const size_t batchSize = 256;
size_t nError = 0;
nnet.setDropout(false);
Batches batches(batchSize, data.size());
for (auto itr = batches.begin(); itr != batches.end(); ++itr) {
auto d = data[itr];
mat x = ~mat(d.x);
mat prob = nnet.feedForward(x);
nError += zeroOneError(prob, d.y);
}
nnet.setDropout(true);
return nError;
}
void AlgorithmRunner::addFavorite(const QString &algName, const DataSet &data) {
if (!PluginLister::pluginExists(QStringToTlpString(algName)))
return;
TulipSettings::instance().addFavoriteAlgorithm(algName);
for (auto i : _favorites) {
if (i->name() == algName)
return;
}
_ui->favoritesBox->widget()->setMinimumHeight(0);
AlgorithmRunnerItem *item = new AlgorithmRunnerItem(algName);
item->setGraph(_graph);
if (!data.empty()) {
item->setData(data);
}
item->setFavorite(true);
int itemPos = 0;
for (auto i : _ui->favoritesBox->widget()->findChildren<AlgorithmRunnerItem *>()) {
if (i->name() > item->name()) {
break;
}
++itemPos;
}
static_cast<QBoxLayout *>(_ui->favoritesBox->widget()->layout())->insertWidget(itemPos, item);
_favorites += item;
item->installEventFilter(this);
item->setAcceptDrops(true);
connect(item, SIGNAL(favorized(bool)), this, SLOT(favorized(bool)));
for (auto i : findChildren<AlgorithmRunnerItem *>()) {
if (i != item && i->name() == algName)
i->setFavorite(true);
}
}
void Toolbox::initWeightsRandomFromMeanAndStd(DataSet &X)
{
//Initialise weights. We use the seed (or clock) to initiliaze random number
//generator
if (seed==0){
srand( (unsigned)time( NULL ) );
}
else {
srand(seed);
}
int nbRawFeatures = X.getNumberofRawFeatures();
//Initialise weights
dVector w(pFeatureGenerator->getNumberOfFeatures());
double widthRangeWeight = fabs(maxRangeWeights - minRangeWeights);
double randValue;
// mean and std_dev
dVector mean(nbRawFeatures);
dVector stdDev(nbRawFeatures);
calculateGlobalMeanAndStd(X,mean,stdDev);
//Initialize weights with global mean and standard deviation
// Only initialize the values especific to the HMM-like HCRF
featureVector* vecFeatures = getAllFeatures(X);
feature* pFeature = vecFeatures->getPtr();
for(int j = 0; j < vecFeatures->size(); j++, pFeature++)
{
switch(pFeature->nodeIndex)
{
case SQUARE_RAW_FEATURE_ID:
randValue = (((double)rand())/(double)RAND_MAX)*2.0*stdDev[(int)pFeature->value]-stdDev[(int)pFeature->value];
w.setValue(pFeature->globalId,mean[(int)pFeature->value] * mean[(int)pFeature->value]+randValue);
break;
case ONE_FEATURE_ID:
case RAW_FEATURE_ID:
randValue = (((double)rand())/(double)RAND_MAX)*2.0*stdDev[(int)pFeature->value]-stdDev[(int)pFeature->value];
w.setValue(pFeature->globalId,mean[(int)pFeature->value]+randValue);
break;
default:
w.setValue(pFeature->globalId,(((double)rand())/(double)RAND_MAX)*widthRangeWeight+minRangeWeights);
}
}
pModel->setWeights(w);
}
/**************************************************************
***
** MaitreyaTextclient --- setLocation
***
***************************************************************/
void setLocation( wxString s )
{
const char MINUS = '-';
Formatter *formatter = Formatter::get();
double lon = 0.0, lat = 0.0, tz = 0.0;
wxString name;
wxStringTokenizer t( s, wxT( " " ));
if ( t.HasMoreTokens()) name = t.GetNextToken();
if ( t.HasMoreTokens())
{
wxString slong = t.GetNextToken();
bool negative = false;
if ( slong.GetChar( 0 ) == '-' )
{
negative = true;
slong = slong.AfterFirst( MINUS );
}
if ( ! formatter->parseDegreeString( slong, lon, 180.0 ))
{
fatalError( "cannot parse longitude string " + slong );
}
if ( negative ) lon *= -1;
printf( "long token %s value %f\n", str2char( slong ), lon );
}
if ( t.HasMoreTokens())
{
wxString slat = t.GetNextToken();
bool negative = false;
if ( slat.GetChar( 0 ) == '-' )
{
negative = true;
slat = slat.AfterFirst( MINUS );
}
if ( ! formatter->parseDegreeString( slat, lat, 90.0 ))
{
fatalError( "cannot parse latitude string " + slat );
}
if ( negative ) lat *= -1;
printf( "lat token %s value %f\n", str2char( slat ), lat );
}
if ( t.HasMoreTokens()) tz = myatof( t.GetNextToken());
dataset.setLocation( name, lon, lat, tz, 0.0 );
printf( "set Location name %s longitude %f latitude %f tz %f\n", str2char( name ), lon, lat, tz );
}
void FrozenDataSet::checkValidFrozenDataSet(const DataSet& dataset) {
// Check that the dataset we have can be converted
PointLayout layout = dataset.layout();
QStringList vlength;
foreach (const QString& name, layout.descriptorNames()) {
if (layout.descriptorLocation(name).lengthType() == VariableLength) vlength << name;
}
if (!vlength.empty()) {
throw GaiaException("The following descriptors are variable-length: ", vlength,
"\nCan only freeze a dataset which is entirely fixed-length at the moment...");
}
if (!layout.descriptorNames(StringType).empty() || !layout.descriptorNames(EnumType).empty()) {
throw GaiaException("Can only freeze datasets which contain only real descriptors (ie: no strings)");
}
}
bool History::addDatasetStandard(const DataSet& _dataset)
{
/// Wait for first ADSC label
if (datasets_standard_[0].data().empty() == true)
{
if (_dataset.label() != datasets_standard_[0].label())
{
return false;
}
}
/// Look for dataset
size_t i = 0;
for (; i < datasets_standard_.max_size(); i++)
{
if (datasets_standard_[i].label() == _dataset.label())
{
break;
}
}
/// Not found ?
if (i == datasets_standard_.max_size()) return false;
/// Check for new data or timestamp
if (datasets_standard_[i].checksum() != _dataset.checksum())
{
/// Update Data
datasets_standard_[i].setData(_dataset.data());
/// Update Timestamp
if (_dataset.timestamp().isTimestamp() == true)
{
datasets_standard_[i].setTimestamp(_dataset.timestamp());
}
/// Update Checksum
datasets_standard_[i].setChecksum(_dataset.checksum());
return true;
}
return false;
}
ReqPtr Rmi::invokeRemoteMethodAsync(DataSet<TextSerializer>& ds)
{
lock_.lock();
if (!serverConn_)
{
lock_.unlock();
// TODO - add error message
return ReqPtr(); // TODO should we return something more intuitive?
}
reqId_++;
if(! ds.add<int>("REQUEST", "REQUESTID", reqId_))
{
std::ostringstream tmp;
tmp << "unable to add REQUEST ID " << reqId_
<< "to section REQUEST" << std::endl;
LOG_ERROR(tmp.str());
return ReqPtr();
}
ReqPtr newRequest(new Request(reqId_)); // will store state of request until reply
// due to asynchronous operation of
// sending
requests_.insert(std::pair<int32_t, ReqPtr>(reqId_,newRequest ));
lock_.unlock();
std::string request = ds.serialize();
//TODO - should this be a part of the serializer itself or probably as a
// o/p packetizer
int32_t reqLength=request.size();
char length[4+1];
memset(length, 0, sizeof(length));
sprintf(length, "%04d", reqLength);
request.insert(0, length);
serverConn_->send(request.c_str(), request.size());
return newRequest; // the client is responsible for calling
// getReply on the new request when he deems proper
}
void
RandomTree<Sample, Label, SplitFunction, SplitEvaluator, LeafNodeStatistics, AppContext>::UpdateLeafStatistics(DataSet<Sample, Label>& dataset)
{
for (int s = 0; s < dataset.size(); s++)
{
// get the current labelled sample
LabelledSample<Sample, Label>* labelled_sample = dataset[s];
// route it to the corresponding leaf node
int node_id = 0;
while(!this->m_nodes[node_id]->m_is_leaf)
{
if (this->m_nodes[node_id]->Split(labelled_sample) == 0)
node_id = (int)m_treetable(node_id, 1);
else
node_id = (int)m_treetable(node_id, 2);
}
// update the leaf node statistics
this->m_nodes[node_id]->UpdateLeafnodeStatistics(labelled_sample);
}
}
TEST(DataSetCoreTest, EditExternalResources)
{
DataSet dataset;
ExternalResource resource("metatype", "id");
resource.Name("file1");
dataset.ExternalResources().Add(resource);
resource.Name("file2").ResourceId("id2");
dataset.ExternalResources().Add(resource);
EXPECT_EQ(2, dataset.ExternalResources().Size());
// edit
dataset.ExternalResources()[0].Name("some new name");
EXPECT_EQ(string("some new name"), dataset.ExternalResources()[0].Name());
EXPECT_EQ(string("file2"), dataset.ExternalResources()[1].Name());
}
void GAB::MiningNeg(int n,DataSet& neg){
const Options& opt = Options::GetInstance();
int pool_size = opt.numThreads;
vector<Mat> region_pool(pool_size);
int st = neg.imgs.size();
int all = 0;
int need = n - st;
double rate;
while(st<n){
#pragma omp parallel for
for(int i = 0;i<pool_size;i++){
region_pool[i] = neg.NextImage(i);
}
#pragma omp parallel for
for (int i = 0; i < pool_size; i++) {
float score = 0;
if(NPDClassify(region_pool[i].clone(),score)){
#pragma omp critical
{
neg.imgs.push_back(region_pool[i].clone());
neg.Fx[st]=score;
if(opt.generate_hd){
char di[256];
sprintf(di,"../data/hd/%d.jpg",st);
imwrite(di,region_pool[i].clone());
}
st++;
}
}
all++;
}
}
neg.size = n;
rate = ((double)(need))/(double)all;
printf("mining success rate %lf\n",rate);
}
DSptr Rmi::invokeRemoteMethod(DataSet<TextSerializer> & ds)
{
lock_.lock();
if (!serverConn_)
{
lock_.unlock();
// TODO add error
return DSptr();
}
reqId_++;
if(! ds.add<int>("REQUEST", "REQUESTID", reqId_))
{
std::ostringstream tmp;
tmp << "unable to add REQUEST ID " << reqId_
<< "to section REQUEST" << std::endl;
LOG_ERROR(tmp.str());
return DSptr();
}
ReqPtr newRequest(new Request(reqId_)); // will store state of request until reply
// due to asynchronous operation of
// sending
requests_.insert(std::pair<int32_t, ReqPtr>(reqId_,newRequest ));
lock_.unlock();
std::string request = ds.serialize();
//TODO - should this be a part of the serializer itself or probably as a
// o/p packetizer
int32_t reqLength=request.size();
char length[4+1];
memset(length, 0, sizeof(length));
sprintf(length, "%04d", reqLength);
request.insert(0, length);
serverConn_->send(request.c_str(), request.size());
return newRequest->getReply(); // will wait until we receive reply or we timeout
}
TEST(DataSetCoreTest, RemoveSubDataSets)
{
DataSet dataset;
EXPECT_EQ(0, dataset.SubDataSets().Size());
DataSetBase sub1;
sub1.Name("subset_1");
DataSetBase sub2;
sub2.Name("subset_2");
dataset.SubDataSets().Add(sub1);
dataset.SubDataSets().Add(sub2);
EXPECT_EQ(2, dataset.SubDataSets().Size());
// remove
dataset.SubDataSets().Remove(sub2);
EXPECT_EQ(1, dataset.SubDataSets().Size());
}
void DataSetTest_GenericData::ColsFromDataSetWithZeroRows()
{
GenericData gd;
GenericFileReader reader;
reader.SetFilename("../data/small_cel_file_with_dataset_of_zero_rows");
reader.ReadHeader(gd);
// Open a DataSet that has 0 rows.
DataSet* ds = gd.DataSet(0,3);
CPPUNIT_ASSERT(ds);
CPPUNIT_ASSERT(ds->Cols() == 2);
CPPUNIT_ASSERT(ds->Rows() == 0);
ds->Delete();
// Open another DataSet that has 0 rows.
ds = gd.DataSet(0,4);
CPPUNIT_ASSERT(ds);
CPPUNIT_ASSERT(ds->Cols() == 2);
CPPUNIT_ASSERT(ds->Rows() == 0);
ds->Delete();
}
// Copy calculated y data to destination collection specified
void DataSpace::copy(Collection* destinationCollection)
{
// Check for valid source and destination collections before we start...
if (!Collection::objectValid(sourceCollection_, "source collection in DataSpace::copy()")) return;
if (!Collection::objectValid(destinationCollection, "destination collection in DataSpace::copy()")) return;
// Clear any existing datasets in the destination collection
destinationCollection->clearDataSets();
// Create destination datasets using those in the sourceCollection_ to get the z values, and size the x/y arrays
for (int n=displayDataSetStart_; n<=displayDataSetEnd_; ++n)
{
DataSet* originalDataSet = sourceCollection_->dataSet(n);
DataSet* newDataSet = destinationCollection->addDataSet();
newDataSet->initialiseData(nPoints_);
destinationCollection->setDataSetZ(newDataSet, originalDataSet->z());
newDataSet->setName("Fit to: "+ originalDataSet->name());
}
// Copy data from DataSpaceRanges/DataSpaceData into the new datasets
for (DataSpaceRange* fitRange = ranges_.first(); fitRange != NULL; fitRange = fitRange->next) fitRange->addCalculatedValues(destinationCollection);
}
static void addStaticDataset(QVector<float>& vals, const QString& name, const DataSet::Type type, const QString& datFileName, Mesh* mesh) {
int nelem = mesh->elements().size();
int nnodes = mesh->nodes().size();
NodeOutput* o = new NodeOutput;
o->init(nnodes, nelem, false);
o->time = 0.0;
o->values = vals;
if (type == DataSet::Bed) {
memset(o->active.data(), 1, nelem); // All cells active
} else {
activateElements(o, mesh);
}
DataSet* ds = new DataSet(datFileName);
ds->setType(type);
ds->setName(name, false);
ds->setIsTimeVarying(false);
ds->addOutput(o); // takes ownership of the Output
ds->updateZRange();
mesh->addDataSet(ds);
}
//
// Draw track line with data labels
//
void ZoomScrollTrack2::trackLineLabel(XYChart *c, int mouseX)
{
// Clear the current dynamic layer and get the DrawArea object to draw on it.
DrawArea *d = c->initDynamicLayer();
// The plot area object
PlotArea *plotArea = c->getPlotArea();
// Get the data x-value that is nearest to the mouse, and find its pixel coordinate.
double xValue = c->getNearestXValue(mouseX);
int xCoor = c->getXCoor(xValue);
// Draw a vertical track line at the x-position
d->vline(plotArea->getTopY(), plotArea->getBottomY(), xCoor,
d->dashLineColor(0x000000, 0x0101));
// Draw a label on the x-axis to show the track line position.
ostringstream xlabel;
xlabel << "<*font,bgColor=000000*> " << c->xAxis()->getFormattedLabel(xValue, "mmm dd, yyyy")
<< " <*/font*>";
TTFText *t = d->text(xlabel.str().c_str(), "arialbd.ttf", 8);
// Restrict the x-pixel position of the label to make sure it stays inside the chart image.
int xLabelPos = max(0, min(xCoor - t->getWidth() / 2, c->getWidth() - t->getWidth()));
t->draw(xLabelPos, plotArea->getBottomY() + 6, 0xffffff);
t->destroy();
// Iterate through all layers to draw the data labels
for (int i = 0; i < c->getLayerCount(); ++i) {
Layer *layer = c->getLayerByZ(i);
// The data array index of the x-value
int xIndex = layer->getXIndexOf(xValue);
// Iterate through all the data sets in the layer
for (int j = 0; j < layer->getDataSetCount(); ++j)
{
DataSet *dataSet = layer->getDataSetByZ(j);
const char *dataSetName = dataSet->getDataName();
// Get the color, name and position of the data label
int color = dataSet->getDataColor();
int yCoor = c->getYCoor(dataSet->getPosition(xIndex), dataSet->getUseYAxis());
// Draw a track dot with a label next to it for visible data points in the plot area
if ((yCoor >= plotArea->getTopY()) && (yCoor <= plotArea->getBottomY()) && (color !=
(int)Chart::Transparent) && dataSetName && *dataSetName)
{
d->circle(xCoor, yCoor, 4, 4, color, color);
ostringstream label;
label << "<*font,bgColor=" << hex << color << "*> "
<< c->formatValue(dataSet->getValue(xIndex), "{value|P4}") << " <*font*>";
t = d->text(label.str().c_str(), "arialbd.ttf", 8);
// Draw the label on the right side of the dot if the mouse is on the left side the
// chart, and vice versa. This ensures the label will not go outside the chart image.
if (xCoor <= (plotArea->getLeftX() + plotArea->getRightX()) / 2)
t->draw(xCoor + 5, yCoor, 0xffffff, Chart::Left);
else
t->draw(xCoor - 5, yCoor, 0xffffff, Chart::Right);
t->destroy();
}
}
}
}
void ViewToolTipAndUrlManager::state(DataSet &data) const {
data.set("Tooltips", _tooltips);
data.set("Url property", _urlPropName);
}
void SimulatedAnnealingOrderTest::test_perform_order_selection(void)
{
message += "test_perform_order_selection\n";
std::string str;
Matrix<double> data;
Vector<Instances::Use> uses;
NeuralNetwork nn;
DataSet ds;
PerformanceFunctional pf(&nn, &ds);
TrainingStrategy ts(&pf);
SimulatedAnnealingOrder sa(&ts);
SimulatedAnnealingOrder::SimulatedAnnealingOrderResults* results;
// Test
str =
"-1 0\n"
"-0.9 0\n"
"-0.8 0\n"
"-0.7 0\n"
"-0.6 0\n"
"-0.5 0\n"
"-0.4 0\n"
"-0.3 0\n"
"-0.2 0\n"
"-0.1 0\n"
"0.0 0\n"
"0.1 0\n"
"0.2 0\n"
"0.3 0\n"
"0.4 0\n"
"0.5 0\n"
"0.6 0\n"
"0.7 0\n"
"0.8 0\n"
"0.9 0\n"
"1 0\n";
data.parse(str);
ds.set(data);
uses.set(21,Instances::Training);
for (size_t i = 0; i < 11; i++)
uses[2*i+1] = Instances::Generalization;
ds.get_instances_pointer()->set_uses(uses);
nn.set(1,3,1);
nn.initialize_parameters(0.0);
pf.set_objective_type(PerformanceFunctional::SUM_SQUARED_ERROR_OBJECTIVE);
ts.set_main_type(TrainingStrategy::QUASI_NEWTON_METHOD);
ts.get_quasi_Newton_method_pointer()->set_display(false);
sa.set_trials_number(1);
sa.set_maximum_order(7);
sa.set_selection_performance_goal(1.0);
sa.set_minimum_temperature(0.0);
sa.set_display(false);
results = sa.perform_order_selection();
assert_true(results->stopping_condition ==
OrderSelectionAlgorithm::SelectionPerformanceGoal, LOG);
// Test
str =
"-1 -1\n"
"-0.9 -0.9\n"
"-0.8 -0.8\n"
"-0.7 -0.7\n"
"-0.6 -0.6\n"
"-0.5 -0.5\n"
"-0.4 -0.4\n"
"-0.3 -0.3\n"
"-0.2 -0.2\n"
"-0.1 -0.1\n"
"0.0 0.0\n"
"0.1 0.1\n"
"0.2 0.2\n"
"0.3 0.3\n"
"0.4 0.4\n"
"0.5 0.5\n"
"0.6 0.6\n"
"0.7 0.7\n"
"0.8 0.8\n"
"0.9 0.9\n"
"1 1\n";
data.parse(str);
ds.set(data);
uses.set(21,Instances::Training);
for (size_t i = 0; i < 11; i++)
uses[2*i+1] = Instances::Generalization;
ds.get_instances_pointer()->set_uses(uses);
nn.set(1,3,1);
nn.initialize_parameters(0.0);
pf.set_objective_type(PerformanceFunctional::SUM_SQUARED_ERROR_OBJECTIVE);
ts.set_main_type(TrainingStrategy::QUASI_NEWTON_METHOD);
ts.get_quasi_Newton_method_pointer()->set_display(false);
sa.set_trials_number(1);
sa.set_maximum_order(7);
sa.set_selection_performance_goal(0.0);
sa.set_minimum_temperature(0.0);
sa.set_display(false);
results = sa.perform_order_selection();
assert_true(results->stopping_condition ==
OrderSelectionAlgorithm::MaximumSelectionFailures, LOG);
}
int main(void)
{
/* First structure and dataset*/
typedef struct s1_t {
int a;
float b;
double c;
} s1_t;
/* Second structure (subset of s1_t) and dataset*/
typedef struct s2_t {
double c;
int a;
} s2_t;
// Try block to detect exceptions raised by any of the calls inside it
try
{
/*
* Initialize the data
*/
int i;
s1_t s1[LENGTH];
for (i = 0; i< LENGTH; i++)
{
s1[i].a = i;
s1[i].b = i*i;
s1[i].c = 1./(i+1);
}
/*
* Turn off the auto-printing when failure occurs so that we can
* handle the errors appropriately
*/
Exception::dontPrint();
/*
* Create the data space.
*/
hsize_t dim[] = {LENGTH}; /* Dataspace dimensions */
DataSpace space( RANK, dim );
/*
* Create the file.
*/
H5File* file = new H5File( FILE_NAME, H5F_ACC_TRUNC );
/*
* Create the memory datatype.
*/
CompType mtype1( sizeof(s1_t) );
mtype1.insertMember( MEMBER1, HOFFSET(s1_t, a), PredType::NATIVE_INT);
mtype1.insertMember( MEMBER3, HOFFSET(s1_t, c), PredType::NATIVE_DOUBLE);
mtype1.insertMember( MEMBER2, HOFFSET(s1_t, b), PredType::NATIVE_FLOAT);
/*
* Create the dataset.
*/
DataSet* dataset;
dataset = new DataSet(file->createDataSet(DATASET_NAME, mtype1, space));
/*
* Write data to the dataset;
*/
dataset->write( s1, mtype1 );
/*
* Release resources
*/
delete dataset;
delete file;
/*
* Open the file and the dataset.
*/
file = new H5File( FILE_NAME, H5F_ACC_RDONLY );
dataset = new DataSet (file->openDataSet( DATASET_NAME ));
/*
* Create a datatype for s2
*/
CompType mtype2( sizeof(s2_t) );
mtype2.insertMember( MEMBER3, HOFFSET(s2_t, c), PredType::NATIVE_DOUBLE);
mtype2.insertMember( MEMBER1, HOFFSET(s2_t, a), PredType::NATIVE_INT);
/*
* Read two fields c and a from s1 dataset. Fields in the file
* are found by their names "c_name" and "a_name".
*/
s2_t s2[LENGTH];
dataset->read( s2, mtype2 );
/*
* Display the fields
*/
cout << endl << "Field c : " << endl;
for( i = 0; i < LENGTH; i++)
cout << s2[i].c << " ";
cout << endl;
cout << endl << "Field a : " << endl;
for( i = 0; i < LENGTH; i++)
cout << s2[i].a << " ";
cout << endl;
/*
* Create a datatype for s3.
*/
CompType mtype3( sizeof(float) );
mtype3.insertMember( MEMBER2, 0, PredType::NATIVE_FLOAT);
/*
* Read field b from s1 dataset. Field in the file is found by its name.
*/
float s3[LENGTH]; // Third "structure" - used to read float field of s1
dataset->read( s3, mtype3 );
/*
* Display the field
*/
cout << endl << "Field b : " << endl;
for( i = 0; i < LENGTH; i++)
cout << s3[i] << " ";
cout << endl;
/*
* Release resources
*/
delete dataset;
delete file;
} // end of try block
// catch failure caused by the H5File operations
catch( FileIException error )
{
error.printErrorStack();
return -1;
}
// catch failure caused by the DataSet operations
catch( DataSetIException error )
{
error.printErrorStack();
return -1;
}
// catch failure caused by the DataSpace operations
catch( DataSpaceIException error )
{
error.printErrorStack();
return -1;
}
// catch failure caused by the DataSpace operations
catch( DataTypeIException error )
{
error.printErrorStack();
return -1;
}
return 0;
}
double* GalacticusReader::readDoubleDataSet(const std::string s, long &nvalues) {
// read a double-type dataset
//std::string s2("Outputs/Output79/nodeData/blackHoleCount");
// DataSet dataset = fp->openDataSet(s);
// rather need pointer to dataset in order to delete it later on:
//cout << "Reading DataSet '" << s << "'" << endl;
DataSet *dptr = new DataSet(fp->openDataSet(s));
DataSet dataset = *dptr; // for convenience
// check class type
H5T_class_t type_class = dataset.getTypeClass();
if (type_class != H5T_FLOAT) {
cout << "Data does not have double type!" << endl;
abort();
}
// check byte order
FloatType intype = dataset.getFloatType();
H5std_string order_string;
H5T_order_t order = intype.getOrder(order_string);
//cout << order_string << endl;
// check again data sizes
if (sizeof(double) != intype.getSize()) {
cout << "Mismatch of double data type." << endl;
abort();
}
size_t dsize = intype.getSize();
//cout << "Data size is " << dsize << endl;
// get dataspace of the dataset (the array length or so)
DataSpace dataspace = dataset.getSpace();
//hid_t dataspace = H5Dget_space(dataset); --> this does not work!! At least not with dataset defined as above!
// get number of dimensions in dataspace
int rank = dataspace.getSimpleExtentNdims();
//cout << "Dataspace rank is " << rank << endl;
// I expect this to be 1 for all Galacticus datasets!
// There are no 2 (or more) dimensional arrays stored in one dataset, are there?
if (rank > 1) {
cout << "ERROR: Cannot cope with multi-dimensional datasets!" << endl;
abort();
}
hsize_t dims_out[1];
int ndims = dataspace.getSimpleExtentDims(dims_out, NULL);
//cout << "dimension " << (unsigned long)(dims_out[0]) << endl;
nvalues = dims_out[0];
// read data
double *buffer = new double[nvalues];
dataset.read(buffer,PredType::NATIVE_DOUBLE);
// the data is stored in buffer now, so we can delete the dataset;
// to do this, call delete on the pointer to the dataset
dataset.close();
delete dptr;
/*cout << "First values: ";
for (int j = 0; j < 10; j++) {
cout << buffer[j] << " ";
}
cout << endl;
*/
DataBlock b;
b.nvalues = nvalues;
b.doubleval = buffer;
b.name = s;
datablocks.push_back(b);
return buffer;
}
long* GalacticusReader::readLongDataSet(const std::string s, long &nvalues) {
// read a long-type dataset
//std::string s2("Outputs/Output79/nodeData/blackHoleCount");
// DataSet dataset = fp->openDataSet(s);
// rather need pointer to dataset in order to delete it later on:
//cout << "Reading DataSet '" << s << "'" << endl;
DataSet *dptr = new DataSet(fp->openDataSet(s)); // need pointer because of "new ..."
DataSet dataset = *dptr; // for convenience
// check class type
H5T_class_t type_class = dataset.getTypeClass();
if (type_class != H5T_INTEGER) {
cout << "Data does not have long type!" << endl;
abort();
}
// check byte order
IntType intype = dataset.getIntType();
H5std_string order_string;
H5T_order_t order = intype.getOrder(order_string);
//cout << order_string << endl;
// check again data sizes
if (sizeof(long) != intype.getSize()) {
cout << "Mismatch of long data type." << endl;
abort();
}
size_t dsize = intype.getSize();
//cout << "Data size is " << dsize << endl;
// get dataspace of the dataset (the array length or so)
DataSpace dataspace = dataset.getSpace();
////hid_t dataspace = H5Dget_space(dataset); --> this does not work!! At least not with dataset defined as above!
// get number of dimensions in dataspace
int rank = dataspace.getSimpleExtentNdims();
//cout << "Dataspace rank is " << rank << endl;
// I expect this to be 1 for all Galacticus datasets!
// There are no 2 (or more) dimensional arrays stored in one dataset, are there?
if (rank > 1) {
cout << "ERROR: Cannot cope with multi-dimensional datasets!" << endl;
abort();
}
hsize_t dims_out[1];
int ndims = dataspace.getSimpleExtentDims(dims_out, NULL);
//cout << "dimension " << (unsigned long)(dims_out[0]) << endl;
nvalues = dims_out[0];
// alternative way of determining data size (needed for buffer memory allocation!)
//size_t size = dataset.getInMemDataSize();
//cout << size << endl;
//int nvalues = size/sizeof(long);
// read data
long *buffer = new long[nvalues]; // = same as malloc
dataset.read(buffer,PredType::NATIVE_LONG);
// the data is stored in buffer now, so we can delete the dataset;
// to do this, call delete on the pointer to the dataset
dataset.close();
// delete dataset is not necessary, if it is a variable on the heap.
// Then it is removed automatically when the function ends.
delete dptr;
//std::vector<int> data_out(NX);
//H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data_out[0]);
// --> this did not work, do not know why.
//cout << "status: " << status << endl;
//int data_out2[dims_out[0]];
//dataset.read(data_out, PredType::NATIVE_LONG, memspace, filespace);
// --> this caused problems with incompatible memspace and filespace etc.
/*cout << "First values: ";
for (int j = 0; j < 10; j++) {
cout << buffer[j] << " ";
}
cout << endl;
*/
DataBlock b;
b.nvalues = nvalues;
b.longval = buffer;
b.name = s;
datablocks.push_back(b);
// b is added to datablocks-vector now
return buffer;
}
void CSVImporter::loadDataSet(DataSetPackage *packageData, const string &locator, boost::function<void(const string &, int)> progressCallback)
{
packageData->isArchive = false;
CSV csv(locator);
csv.open();
vector<string> columns = vector<string>();
vector<vector<string> > cells = vector<vector<string> >();
csv.readLine(columns);
unsigned long long progress;
unsigned long long lastProgress = -1;
size_t columnCount = columns.size();
for (size_t i = 0; i < columnCount; i++) // columns
cells.push_back(vector<string>());
vector<string> line;
bool success = csv.readLine(line);
while (success)
{
progress = 50 * csv.pos() / csv.size();
if (progress != lastProgress)
{
progressCallback("Loading Data Set", progress);
lastProgress = progress;
}
if (line.size() != 0) {
size_t i = 0;
for (; i < line.size() && i < columnCount; i++)
cells[i].push_back(line[i]);
for (; i < columnCount; i++)
cells[i].push_back(string());
}
line.clear();
success = csv.readLine(line);
}
packageData->dataSet = SharedMemory::createDataSet(); // this is required incase the loading of the data fails so that the SharedMemory::createDataSet() can be later freed.
do
{
try {
success = true;
DataSet *dataSet = packageData->dataSet;
dataSet->setColumnCount(columnCount);
if (cells.size() > 0)
dataSet->setRowCount(cells.at(0).size());
}
catch (boost::interprocess::bad_alloc &e)
{
try {
packageData->dataSet = SharedMemory::enlargeDataSet(packageData->dataSet);
success = false;
}
catch (exception &e)
{
throw runtime_error("Out of memory: this data set is too large for your computer's available memory");
}
}
catch (exception &e)
{
cout << "n " << e.what() << "\n";
cout.flush();
}
catch (...)
{
cout << "something else\n ";
cout.flush();
}
}
while ( ! success);
for (int colNo = 0; colNo < packageData->dataSet->columnCount(); colNo++)
{
bool success;
do {
success = true;
try {
DataSet *dataSet = packageData->dataSet;
progressCallback("Loading Data Set", 50 + 50 * colNo / dataSet->columnCount());
string columnName = columns.at(colNo);
if (columnName == "")
{
stringstream ss;
ss << "V";
ss << (colNo + 1);
columnName = ss.str();
}
Column &column = dataSet->column(colNo);
initColumn(column, columnName, cells.at(colNo));
}
catch (boost::interprocess::bad_alloc &e)
{
try {
packageData->dataSet = SharedMemory::enlargeDataSet(packageData->dataSet);
success = false;
}
catch (exception &e)
{
throw runtime_error("Out of memory: this data set is too large for your computer's available memory");
}
}
catch (exception e)
{
cout << "n " << e.what();
cout.flush();
}
catch (...)
{
cout << "something else\n ";
cout.flush();
}
} while (success == false);
}
}
int main(int argc, const char *argv[])
{
using std::string;
using std::cerr;
using std::cout;
using std::endl;
using std::valarray;
using namespace NonmemPars;
// numberEval
valarray<int> numberEval = MontePars::numberEval;
// method
std::string MethodName;
bool analytic = false;
bool grid = false;
bool monte = false;
switch( MontePars::method )
{
case MontePars::analytic:
analytic = true;
MethodName = "analytic";
break;
case MontePars::grid:
grid = true;
MethodName = "grid";
break;
case MontePars::plain:
monte = true;
MethodName = "plain";
break;
case MontePars::miser:
monte = true;
MethodName = "miser";
break;
default:
cerr << "monteDriver: ";
cerr << "method is no analytic, grid, plain, or miser" << endl;
return ReturnFailure;
}
if( analytic && NonmemPars::nEta != 1 )
{ cerr << "monteDriver: ";
cerr << "method is analytic and nEta != 1" << endl;
return ReturnFailure;
}
size_t i;
for(i = 0; i < numberEval.size(); i++)
{ if( numberEval[i] <= 0 )
{ cerr << "monteDriver: ";
cerr << "numberEval is not greater than zero" << endl;
return ReturnFailure;
}
}
// data set
DataSet< CppAD::AD<double> > set;
Pred< CppAD::AD<double> > mPred(&set);
const int nPop = set.getPopSize();
if( nPop <= 0 )
{ cerr << "monteDriver: DataSet.getPopSize() <= 0 " << endl;
return ReturnFailure;
}
valarray<int> N = set.getN();
for(i = 0; i < nPop; i++)
{ if( N[i] <= 0 )
{ cerr << "monteDriver: DataSet.getN() <= 0" << endl;
return ReturnFailure;
}
}
valarray<double> y = set.getAllMeasurements();
const int nY = N.sum();
if( nY != y.size() )
{ cerr << "monteDriver: y.size != N[0] + ... + N[M-1]" << endl;
return ReturnFailure;
}
// model constructor
PopPredModel model(
mPred,
nTheta,
thetaLow,
thetaUp,
thetaIn,
nEta,
etaIn,
nEps,
omegaStruct,
omegaIn,
sigmaStruct,
sigmaIn
);
// get the input value for the fixed effects as a single vector
const int nAlp = model.getNPopPar();
valarray<double> alpIn (nAlp);
model.getPopPar( alpIn );
// get the limits on the fixed effects
valarray<double> alpLow(nAlp);
valarray<double> alpUp(nAlp);
model.getPopParLimits(alpLow, alpUp);
// step size in fixed effects
valarray<double> alpStep(nAlp);
alpStep = 2e-2 * (alpUp - alpLow);
// get the limits on the random effects
const int nB = model.getNIndPar();
valarray<double> bLow( nB );
valarray<double> bUp( nB );
model.getIndParLimits( bLow, bUp );
// start the output file
cout << "<?xml version=\"1.0\"?>" << endl;
cout << "<spkreport>" << endl;
// start timing
timeval timeBegin;
gettimeofday( &timeBegin, NULL );
// space used to hold the results
double pop_obj_estimate;
double pop_obj_stderror;
valarray<double> obj_value(nAlp * 3);
valarray<double> obj_std(nAlp * 3);
try
{ // loop over two indices in fixed effects vector
size_t index;
valarray<double> alp (nAlp);
for(i = 0; i < nAlp; i++)
{ int m;
for(m = 0; m < 3; m++) if( i == 0 || m != 1 )
{ double step = (m-1) * alpStep[i];
alp = alpIn;
alp[i] = alp[i] + step;
// analytic integral
if( analytic ) AnalyticIntegralAll(
pop_obj_estimate,
pop_obj_stderror,
model ,
N ,
y ,
alp ,
bLow ,
bUp
);
// grid integral approximation
if( grid ) GridIntegralAll(
pop_obj_estimate,
pop_obj_stderror,
model ,
N ,
y ,
alp ,
bLow ,
bUp ,
numberEval
);
// Monte Carlo integral approximation
if( monte ) MonteIntegralAll(
pop_obj_estimate,
pop_obj_stderror,
model ,
N ,
y ,
alp ,
bLow ,
bUp ,
numberEval
);
// save results
index = i * 3 + m;
obj_value[index] = pop_obj_estimate;
obj_std[index] = pop_obj_stderror;
}
// if m == 1 then result is the same for all i
index = i * 3 + 1;
obj_value[index] = obj_value[1];
obj_std[index] = obj_std[1];
}
}
catch( const SpkException& e )
{ cout << "<error_message>" << endl;
cout << e << endl;
cout << "</error_message>" << endl;
return ReturnSuccess;
}
catch( ... )
{ cout << "<error_message>" << endl;
cout << "Unknown exception occurred";
cout << "</error_message>" << endl;
return ReturnSuccess;
}
timeval timeEnd;
gettimeofday( &timeEnd, NULL );
// report the time in seconds that Monte Carlo integration required
double pop_obj_seconds = difftime(timeEnd.tv_sec, timeBegin.tv_sec );
cout << "<pop_monte_result seconds=\"" << pop_obj_seconds
<< "\" method=\"" << MethodName
<< "\" numberEval=\"" << numberEval[0];
for(i = 1; i < numberEval.size(); i++)
cout << ", " << numberEval[i];
cout << "\" >" << endl;
size_t indent = 4;
size_t nrows = nAlp;
size_t ncols = 1;
OutputColumnMajor(indent, alpIn, "alpha_center", nrows, ncols);
OutputColumnMajor(indent, alpStep, "alpha_step", nrows, ncols);
nrows = nAlp;
ncols = 3;
OutputRowMajor(indent, obj_value, "obj_value", nrows, ncols);
OutputRowMajor(indent, obj_std, "obj_std", nrows, ncols);
// return from main program
cout << "</pop_monte_result>" << endl;
cout << "</spkreport>" << endl;
return ReturnSuccess;
}
void KWIKFileSource::fillRecordInfo()
{
Group recordings;
try
{
recordings = sourceFile->openGroup("/recordings");
int numObjs = recordings.getNumObjs();
for (int i=0; i < numObjs; i++)
{
try
{
Group recordN;
DataSet data;
Attribute attr;
DataSpace dSpace;
float sampleRate;
float bitVolts;
hsize_t dims[3];
RecordInfo info;
recordN = recordings.openGroup(String(i).toUTF8());
data = recordN.openDataSet("data");
attr = recordN.openAttribute("sample_rate");
attr.read(PredType::NATIVE_FLOAT,&sampleRate);
attr = recordN.openAttribute("bit_depth");
attr.read(PredType::NATIVE_FLOAT,&bitVolts);
dSpace = data.getSpace();
dSpace.getSimpleExtentDims(dims);
info.name="Record "+String(i);
info.numSamples = dims[0];
info.sampleRate = sampleRate;
bool foundBitVoltArray = false;
HeapBlock<float> bitVoltArray(dims[1]);
try
{
recordN = recordings.openGroup((String(i) + "/application_data").toUTF8());
attr=recordN.openAttribute("channel_bit_volts");
attr.read(ArrayType(PredType::NATIVE_FLOAT,1,&dims[1]),bitVoltArray);
foundBitVoltArray = true;
} catch (GroupIException)
{
} catch (AttributeIException)
{
}
for (int j = 0; j < dims[1]; j++)
{
RecordedChannelInfo c;
c.name = "CH" + String(j);
if (foundBitVoltArray)
c.bitVolts = bitVoltArray[j];
else
c.bitVolts = bitVolts;
info.channels.add(c);
}
infoArray.add(info);
availableDataSets.add(i);
numRecords++;
}
catch (GroupIException)
{
}
catch (DataSetIException)
{
}
catch (AttributeIException)
{
}
catch (DataSpaceIException error)
{
PROCESS_ERROR;
}
}
}
catch (FileIException error)
{
PROCESS_ERROR;
}
catch (GroupIException error)
{
PROCESS_ERROR;
}
}
void SetUp() override {
dataSet.load_from_file(DATASET_DIR "test_dataset.txt");
}
bool SplitEvaluatorMLRegr<Sample>::DoFurtherSplitting(DataSet<Sample, LabelMLRegr>& dataset, int depth)
{
if (depth >= (this->m_appcontext->max_tree_depth-1) || (int)dataset.size() < this->m_appcontext->min_split_samples)
return false;
return true;
}
/** Set up histogram with specified data sets. */
Analysis::RetType Analysis_Hist::Setup(ArgList& analyzeArgs, AnalysisSetup& setup, int debugIn)
{
debug_ = debugIn;
// Keywords
std::string histname = analyzeArgs.GetStringKey("name");
outfilename_ = analyzeArgs.GetStringKey("out");
if (outfilename_.empty()) {
mprinterr("Error: Hist: No output filename specified.\n");
return Analysis::ERR;
}
traj3dName_ = analyzeArgs.GetStringKey("traj3d");
traj3dFmt_ = TrajectoryFile::WriteFormatFromString( analyzeArgs.GetStringKey("trajfmt"),
TrajectoryFile::AMBERTRAJ );
parmoutName_ = analyzeArgs.GetStringKey("parmout");
// Create a DataFile here so any DataFile arguments can be processed. If it
// turns out later that native output is needed the DataFile will be removed.
outfile_ = setup.DFL().AddDataFile(outfilename_, analyzeArgs);
if (outfile_==0) return Analysis::ERR;
Temp_ = analyzeArgs.getKeyDouble("free",-1.0);
if (Temp_!=-1.0)
calcFreeE_ = true;
else
calcFreeE_ = false;
gnuplot_ = analyzeArgs.hasKey("gnu");
if (analyzeArgs.hasKey("norm"))
normalize_ = NORM_SUM;
else if (analyzeArgs.hasKey("normint"))
normalize_ = NORM_INT;
else
normalize_ = NO_NORM;
circular_ = analyzeArgs.hasKey("circular");
nativeOut_ = analyzeArgs.hasKey("nativeout");
if ( analyzeArgs.Contains("min") ) {
default_min_ = analyzeArgs.getKeyDouble("min", 0.0);
minArgSet_ = true;
}
if ( analyzeArgs.Contains("max") ) {
default_max_ = analyzeArgs.getKeyDouble("max", 0.0);
maxArgSet_ = true;
}
default_step_ = analyzeArgs.getKeyDouble("step", 0.0) ;
default_bins_ = analyzeArgs.getKeyInt("bins", -1);
calcAMD_ = false;
std::string amdname = analyzeArgs.GetStringKey("amd");
if (!amdname.empty()) {
DataSet* ds = setup.DSL().GetDataSet( amdname );
if (ds == 0) {
mprinterr("Error: AMD data set %s not found.\n", amdname.c_str());
return Analysis::ERR;
}
if (ds->Ndim() != 1) {
mprinterr("Error: AMD data set must be 1D.\n");
return Analysis::ERR;
}
amddata_ = (DataSet_1D*)ds;
calcAMD_ = true;
}
// Treat all remaining arguments as dataset names. Do not set up dimensions
// yet since the data sets may not be fully populated.
ArgList dsetNames = analyzeArgs.RemainingArgs();
for ( ArgList::const_iterator setname = dsetNames.begin();
setname != dsetNames.end(); ++setname)
{
if (CheckDimension( *setname, setup.DSL() )) return Analysis::ERR;
}
// histdata contains the DataSets to be histogrammed
if (histdata_.empty()) {
mprinterr("Error: Hist: No datasets specified.\n");
return Analysis::ERR;
}
// Total # of dimensions for the histogram is the number of sets to be binned.
N_dimensions_ = histdata_.size();
if (!nativeOut_) {
switch ( N_dimensions_ ) {
case 1: hist_ = setup.DSL().AddSet( DataSet::DOUBLE, histname, "Hist"); break;
case 2: hist_ = setup.DSL().AddSet( DataSet::MATRIX_DBL, histname, "Hist"); break;
// TODO: GRID_DBL
case 3: hist_ = setup.DSL().AddSet( DataSet::GRID_FLT, histname, "Hist"); break;
default: // FIXME: GET N DIMENSION CASE!
mprintf("Warning: Histogram dimension > 3. DataSet/DataFile output not supported.\n");
nativeOut_ = true;
}
}
// traj3d only supported with 3D histograms
if (!traj3dName_.empty() && N_dimensions_ != 3) {
mprintf("Warning: 'traj3d' only supported with 3D histograms.\n");
traj3dName_.clear();
parmoutName_.clear();
}
if (!nativeOut_) {
// DataFile output. Add DataSet to DataFile.
if (hist_ == 0) {
mprinterr("Error: Could not set up histogram data set.\n");
return Analysis::ERR;
}
outfile_->AddDataSet( hist_ );
} else {
// Native output. Remove DataFile from DataFileList
outfile_ = setup.DFL().RemoveDataFile( outfile_ );
native_ = setup.DFL().AddCpptrajFile( outfilename_, "Histogram output" );
if (native_ == 0) return Analysis::ERR;
}
mprintf("\tHist: %s: Set up for %zu dimensions using the following datasets:\n",
outfilename_.c_str(), N_dimensions_);
mprintf("\t[ ");
for (std::vector<DataSet_1D*>::iterator ds=histdata_.begin(); ds!=histdata_.end(); ++ds)
mprintf("%s ",(*ds)->legend());
mprintf("]\n");
if (calcAMD_)
mprintf("\tPopulating bins using AMD boost from data set %s\n",
amddata_->legend());
if (calcFreeE_)
mprintf("\tFree energy in kcal/mol will be calculated from bin populations at %f K.\n",Temp_);
if (nativeOut_)
mprintf("\tUsing internal routine for output. Data will not be stored on the data set list.\n");
//if (circular_ || gnuplot_) {
// mprintf("\tWarning: gnuplot and/or circular specified; advanced grace/gnuplot\n");
// mprintf("\t formatting disabled.\n");*/
if (circular_)
mprintf("\tcircular: Output coordinates will be wrapped.\n");
if (gnuplot_ && outfile_ == 0)
mprintf("\tgnuplot: Output will be in gnuplot-readable format.\n");
//}
if (normalize_ == NORM_SUM)
mprintf("\tnorm: Sum over bins will be normalized to 1.0.\n");
else if (normalize_ == NORM_INT)
mprintf("\tnormint: Integral over bins will be normalized to 1.0.\n");
if (!traj3dName_.empty()) {
mprintf("\tPseudo-trajectory will be written to '%s' with format %s\n",
traj3dName_.c_str(), TrajectoryFile::FormatString(traj3dFmt_));
if (!parmoutName_.empty())
mprintf("\tCorresponding pseudo-topology will be written to '%s'\n",
parmoutName_.c_str());
}
return Analysis::OK;
}
DataSet *load_SS_fMRI_DS(std::string pathname){
std::cout << "Loading fMRI" << std::endl;
std::string filename, datapath;
struct dirent *filep;
struct stat filestat;
std::ifstream file;
DIR *dir;
std::string path;
Matrix data(220, 63*53);
for (int s = 1; s <=1; ++s) {
std::stringstream n;
n << s;
path = pathname + n.str() + "/";
dir = opendir(path.c_str());
Matrix subject(220, 63*53);
int sample = 0;
while ((filep = readdir(dir))){
filename = filep->d_name;
datapath = path + filep->d_name;
// If the file is a directory (or is in some way invalid) we'll skip it
if (stat( datapath.c_str(), &filestat )) continue;
if (S_ISDIR( filestat.st_mode )) continue;
if (filename == ".DS_Store") continue;
//cout << "Loading " << filename << endl;
file.open(datapath.c_str());
std::string line;
int index = 0;
while (getline(file, line)){
float value;
std::istringstream iss(line);
while (iss >> value) {
subject(sample, index) = value;
++index;
}
}
file.close();
++sample;
}
closedir(dir);
//subject.normalize();
data.fill_submatrix(subject, (s-1)*200);
}
Vector meanImage = data.mean_image();
Vector mask = meanImage.make_mask();
data.remove_mask(mask);
DataSet *dataset = new DataSet(SSL_VIS, 220, 53, 63, 1, data.dim2);
dataset->data_path = pathname;
dataset->data = data;
dataset->mask = mask;
Vector mi = dataset->data.sd_image();
dataset->zeromean_unitvar_pixel();
dataset->applymask = true;
return dataset;
}
int GalacticusReader::readNextBlock(string outputName) {
// read one complete Output* block from Galacticus HDF5-file
// should fit into memory ... if not, need to adjust this
// and provide the number of values to be read each time
long nvalues;
//char outputname[1000];
//performance output stuff
boost::posix_time::ptime startTime;
boost::posix_time::ptime endTime;
string newtext = "";
boost::regex re(":z[0-9.]*");
startTime = boost::posix_time::microsec_clock::universal_time();
// first get names of all DataSets in nodeData group and their item size
//cout << "outputName: " << outputName<< endl;
Group group(fp->openGroup(outputName));
// maybe check here that it worked?
hsize_t len = group.getNumObjs();
//cout << "Iterating over Datasets in the group ... " << endl;
//H5L_iterate_t
//vector<string> dsnames;
dataSetNames.clear(); // actually, the names should be exactly the same as for the group before!! --> check this???
int idx2 = H5Literate(group.getId(), H5_INDEX_NAME, H5_ITER_INC, NULL, file_info, &dataSetNames);
string s;
string dsname;
string matchname;
int numDataSets = dataSetNames.size();
//cout << "numDataSets: " << numDataSets << endl;
// create a key-value map for the dataset names, do it from scratch for each block,
// and remove redshifts from the dataset names (where necessary)
dataSetMap.clear();
for (int k=0; k<numDataSets; k++) {
dsname = dataSetNames[k];
// convert to matchname, i.e. remove possibly given redshift from the name:
string matchname = boost::regex_replace(dsname, re, newtext);
dataSetMap[matchname] = k;
//dataSetMatchNames.push_back(matchname);
}
// clear datablocks from previous block, before reading new ones:
datablocks.clear();
// read each desired data set, use corresponding read routine for different types
for (int k=0; k<numDataSets; k++) {
dsname = dataSetNames[k];
s = string(outputName) + string("/") + dsname;
DataSet *dptr = new DataSet(fp->openDataSet(s));
DataSet dataset = *dptr; // for convenience
// check class type
H5T_class_t type_class = dataset.getTypeClass();
if (type_class == H5T_INTEGER) {
//cout << "DataSet has long type!" << endl;
long *data = readLongDataSet(s, nvalues);
} else if (type_class == H5T_FLOAT) {
//cout << "DataSet has double type!" << endl;
double *data2 = readDoubleDataSet(s, nvalues);
}
//cout << nvalues << " values read." << endl;
}
// How to proceed from here onwards??
// Could read all data into data[0] to data[104] or so,
// but I need to keep the information which is which!
// Alternatively create one big structure to hold it all?
// => use a small class that contains
// 1) name of dataset
// 2) array of values, number of values
// use vector<newclass> to create a vector of these datasets.
// maybe can use datasets themselves, so no need to define own class?
// => assigning to the new class has already happened now inside the read-class.
endTime = boost::posix_time::microsec_clock::universal_time();
printf("Time for reading output %s (%ld rows): %lld ms\n", outputName.c_str(), nvalues, (long long int) (endTime-startTime).total_milliseconds());
fflush(stdout);
return nvalues; //assume that nvalues is the same for each dataset (datablock) inside one Output-group (same redshift)
}
