/// random read from nxn array. all indices are read once
void randomRead()
{
i64 k[n*n];
for (int i = 0; i < n*n; i++)
{
k[i] = i;
}
MDArray *array = new MDArray(fileName);
PagedStorageContainer::resetPerfCounts();
permute(k, n*n);
gettimeofday(&tim, NULL);
t1 = tim.tv_sec + tim.tv_usec/1000000.0;
for (int i = 0; i < n*n; i++)
{
coord = MDCoord(2, k[i]/n, k[i]%n);
array->get(coord, datum);
}
delete array;
gettimeofday(&tim, NULL);
t2 = tim.tv_sec + tim.tv_usec/1000000.0;
readCount += PagedStorageContainer::readCount;
writeCount += PagedStorageContainer::writeCount;
accessTime += PagedStorageContainer::accessTime;
execTime += (t2 - t1);
fprintf(pFile, "%-14s %8i %8i %11.3f %11.3f\n", "rand read", PagedStorageContainer::readCount, PagedStorageContainer::writeCount, PagedStorageContainer::accessTime, t2-t1);
}
TEST(MDArray, Read)
{
i64 rows = 20L;
i64 cols = 20L;
MDCoord dim(2, rows, cols);
StorageType type = DMA;
i64 blockDims[] = {10L, 10L};
u8 blockOrders[] = {0, 1};
u8 microOrders[] = {0, 1};
//permute(blockOrders, dim.nDim);
//permute(microOrders, dim.nDim);
Linearization *block = new BlockBased(dim.nDim, dim.coords, blockDims,
blockOrders, microOrders);
const char *fileName = "test1.bin";
MDCoord key;
Datum_t datum = 1.0;
MDArray *array = new MDArray(dim, type, block, fileName);
MDIterator *it = array->createIterator(Dense, block);
srand(time(NULL));
const int num=100;
std::map<int, MDCoord> keys;
std::map<int, Datum_t> data;
int total = rows*cols;
for(int i=0; i<num; i++) {
int k = rand() % total;
if (data.find(k) == data.end()) {
data[k] = rand();
it->setIndexRange(k, k+1);
it->moveNext();
it->put(data[k]);
MDCoord key;
it->get(key, datum);
keys[k] = key;
}
else
i--;
}
delete it;
delete array;
array = new MDArray(fileName);
Datum_t datum1;
for (std::map<int,MDCoord>::iterator s = keys.begin();
s != keys.end();
++s) {
//cout<<"checking "<<s->second.toString()<<endl;
array->get(s->second, datum1);
ASSERT_DOUBLE_EQ(data[s->first], datum1);
}
delete array;
delete block;
}
void checkRect(const MDArray<2> &array, const MDCoord<2> &begin, const MDCoord<2> &end, const Datum_t *data)
{
int k = 0;
Datum_t datum;
for (i64 j=begin[1]; j<=end[1]; ++j)
for (i64 i=begin[0]; i<=end[0]; ++i) {
array.get(MDCoord<2>(i,j), datum);
ASSERT_EQ(data[k++], datum);
}
}
void printMDArray(const MDArray<nDim> &array)
{
MDCoord<nDim> dim = array.getDims();
Datum_t datum;
for (int i=0; i<dim[0]; i++) {
for (int j=0; j<dim[1]; j++) {
MDCoord<2> c(i,j);
array.get(c, datum);
cout<<datum<<"\t";
}
cout<<endl;
}
}
void ForwardBacktracker::backtrack_pass(uint start, uint end, uint seq_len, MDArray<eMISMASK> & mismask, RandomGen* rg) {
uint choice = 0;
MDArray<double> transition_matrix(vec(hidden_node_size));
// The backtrack loop
for(int l=end-1; l >= ((int) start); l--) {
uint i = l - start;
DiscreteNode *node;
// Setup the node, cpd and nodes
if(i==0) {
node = hd_0;
}
else {
node = hd_1;
}
// Choose whether to sample from the forward probability
// distribution or take the previous choice into account
if ( ((uint) l)==end-1 ) {
transition_matrix = forward.get_view(i);
}
else {
// TODO: Make this sum faster
for (uint j=0; j<hidden_node_size; j++)
transition_matrix[j] = forward.get(i,j) * transition_matrices[i+1]->get(j, choice);
transition_matrix.normalize();
}
// Sample the hidden node from the forward probability distribution
transition_matrix.cumsum();
assert(rg);
double r = rg->get_rand();
choice = transition_matrix.bisect(r);
node->parentmap.set(l, choice);
// Sample the children
for(vector<Node*>::iterator child=node->children_1.begin(); child < node->children_1.end(); child++) {
if (mismask.get(l, (*child)->node_index) != MOCAPY_OBSERVED) {
(*child)->sample(l);
}
}
}
}
void Shrinkage::generate_wk(MDArray<double> mat,int SDM_type)
{
int n=mat.get_shape()[0];
int p=mat.get_shape()[1];
vector<uint> index;
double sum,term1,term2;
MDArray<double> sub_means(get_sub_means(mat));
for(int i=0;i<p;i++)
for(int j=0;j<p;j++){
sum=0;
for(int k=0;k<n;k++){
term1=mat.get(k,i)-sub_means[i];
term2=mat.get(k,j)-sub_means[j];
index=vec((uint)k, (uint)i, (uint)j);
if(SDM_type)
wk_StDM[index]=term1*term2;
else
wk[index]=term1*term2;
}
}
}
TEST(DISABLED_MDArray, Create)
{
i64 rows = 500L;
i64 cols = 500L;
MDCoord dim(2, rows, cols);
StorageType type = DMA;
Linearization *row = new RowMajor(dim);
Linearization *col = new ColMajor(dim);
i64 blockDims[] = {100L, 100L};
u8 blockOrders[] = {0, 1};
u8 microOrders[] = {0, 1};
permute(blockOrders, dim.nDim);
permute(microOrders, dim.nDim);
Linearization *block = new BlockBased(dim.nDim, dim.coords, blockDims,
blockOrders, microOrders);
const char *fileName = "test.bin";
MDArray *array = new MDArray(dim, type, row, fileName);
MDCoord coord1 = MDCoord(2, 120, 19);
ASSERT_TRUE(AC_OK == array->put(coord1, 12345));
Datum_t datum;
ASSERT_TRUE(AC_OK == array->get(coord1, datum));
ASSERT_DOUBLE_EQ(12345, datum);
MDCoord coord2 = MDCoord(2, 400, 133);
ASSERT_TRUE(AC_OK == array->put(coord2, 12.345));
MDCoord coord3 = MDCoord(2, 256, 19);
ASSERT_TRUE(AC_OK == array->put(coord3, 1000));
ASSERT_TRUE(AC_OK == array->get(coord2, datum));
ASSERT_DOUBLE_EQ(12.345, datum);
ASSERT_TRUE(AC_OK == array->get(coord3, datum));
ASSERT_DOUBLE_EQ(1000, datum);
// dense itor, write
MDIterator *it = array->createIterator(Dense, row);
MDCoord coord;
int i = 0;
while (it->moveNext())
{
it->put(i);
i++;
}
ASSERT_EQ(rows*cols, i);
delete it;
// block itor, read
it = array->createIterator(Dense, block);
i = 0;
while(it->moveNext())
{
it->get(coord, datum);
ASSERT_DOUBLE_EQ(row->linearize(coord), datum);
i++;
}
ASSERT_EQ(rows*cols, i);
delete it;
/*
// natural itor, read
it = array->createNaturalIterator(Dense);
i = 1;
while(it->moveNext())
{
it->get(coord, datum);
ASSERT_EQ(i, row->linearize(coord));
ASSERT_EQ(i, datum);
i++;
}
ASSERT_EQ(rows*cols, i);
delete it;
*/
delete row;
delete col;
delete block;
delete array;
}
void ForwardBacktracker::forward_pass(uint start, uint end, uint seq_len, MDArray<eMISMASK> & mismask, bool multiply_by_parents) {
// Set variables used in the forward algorithm
uint slice_count = end - start;
// Setup the forward array and the scales
scales.clear();
scales.reserve(slice_count);
forward.set_shape(vec(slice_count, hidden_node_size));
transition_matrices.clear();
transition_matrices.reserve(slice_count);
// Fill out the forward array
for(uint i=0, l=start; i < slice_count; i++, l++) {
DiscreteNode *node;
MDArray<double> *cpd;
vector<Node*> *nodes;
// For the first slice in the network
if(l==0 || i==0) {
// Get the appropriate hidden node
if(l==0) {
node = hd_0;
cpd = cpd_0;
}
else {
node = hd_1;
cpd = cpd_1;
}
// Get the values of the parents of the hidden node
// (and remove the value of the hidden node)
vector<double> dpv;
vector<uint> ipv;
node->parentmap.get(l, dpv);
dpv.pop_back();
toint(dpv, ipv);
// Get the transition probability and store it in the forward array
MDArray<double> *cpd_sliced = &cpd->get_view(ipv);
forward.set(i, *cpd_sliced);
// Store the transition matrix from a posible backtrack
transition_matrices.push_back(cpd_sliced);
}
// For the remaining slices
else {
// Get the appropriate hidden node
node = hd_1;
// Get the values of the parents of the hidden node
// (and remove the value of the hidden node)
vector<double> dpv;
vector<uint> ipv;
node->parentmap.get(l, dpv);
dpv.pop_back();
dpv.erase(dpv.begin());
toint(dpv, ipv);
MDArray<double> *cpd_sliced = &cpd_1_swaped.get_view(ipv);
// Store the transition matrix from a posible backtrack
transition_matrices.push_back(cpd_sliced);
// Multiply transitions probabilities P(HD_l = g | HD_{l-1} = h ) by the
// forward values forward[i-1, g] and sum over node values of g
// TODO: This can probably be done faster
for(uint j=0; j < hidden_node_size; j++ ) {
// Sum over the previous hidden value
double sum = 0;
for(uint k=0; k < hidden_node_size; k++ ) {
sum += forward.get(i-1, k) * cpd_sliced->get(k, j);
}
forward.set(i, j, sum);
}
}
// Multiply the forward value by the probability of the parents
if(multiply_by_parents) {
if(l==0)
nodes = &nodes_0;
else
nodes = &nodes_1;
for(vector<uint>::iterator parent=node->parents_1.begin(); parent < node->parents_1.end(); parent++) {
double parent_likelihood = exp( (*nodes)[(*parent)]->get_slice_log_likelihood(l) );
forward.get_view(i).multiply_inplace(parent_likelihood);
}
}
// Multiply the forward values by the probability of the observed children
// Note that the value of the hidden node is preserved in prev_node_value
for(vector<Node*>::iterator child=node->children_1.begin(); child < node->children_1.end(); child++) {
if (mismask.get(l, (*child)->node_index) == MOCAPY_OBSERVED) {
// Get the original value of the hidden node
vector<double> dpv;
node->parentmap.get(l, dpv);
// Multiply the forward vector with the likelihood values of the child
for(uint j=0; j < hidden_node_size; j++ ) {
node->parentmap.set(l, j);
forward.get(i,j) *= exp( (*child)->get_slice_log_likelihood(l) );
}
// Restore the original value of the hidden node
node->parentmap.set(l, dpv.back());
}
}
// If the is the end slice but not the end of the DBN, take the next hidden value into account
if(l==end-1 && l!=seq_len-1) {
// The next node can only be hd_1
vector<double> dpv;
vector<uint> ipv;
hd_1->parentmap.get(l+1, dpv);
uint next_hidden_value = (uint)dpv.back();
dpv.pop_back();
dpv.erase(dpv.begin());
toint(dpv, ipv);
MDArray<double> * cpd_1_sliced = &cpd_1_swaped.get_view(ipv);
// Multiply the forward vector with the probability of the transition to the next hidden state
for(uint j=0; j < hidden_node_size; j++ ) {
forward.get(i,j) *= cpd_1_sliced->get(j, next_hidden_value);
}
}
// Scale forward to be a probability distribution (this is
// allowed according to BSA sec. 3.6 (p. 78)
vector<double> *column_i = (&(&forward.get_view(i))->get_values());
double scale = 0;
// Sum column i
for(uint j=0; j < hidden_node_size; j++ ) {scale += (*column_i)[j];}
// Normalise column i by it's sum
bool observedNan=false;
for(uint j=0; j < hidden_node_size; j++ ) {
(*column_i)[j] /= scale;
if (isnan((*column_i)[j]))
observedNan = true;
}
if (observedNan) {
for(uint j=0; j < hidden_node_size; j++ ) {
(*column_i)[j] = 1.0/hidden_node_size;
}
}
// Save the scale
scales.push_back(scale);
}
}
