TEST(SegmentArray, LargeArray) {
// This test is named "Negative", and belongs to the "FactorialTest"
// test case.
NeUseDevice(&gDevProp);
int bigsize = gDevProp.totalGlobalMem/(sizeof(int) * 20);
CudppPlanFactory planPool;
SegmentArray<int> a(&planPool, bigsize);
SegmentArray<int> zeroArray(&planPool, 0);
EXPECT_EQ(1, a.getNumSegments());
EXPECT_EQ(a.getSize(), a.getSegmentLength(0));
EXPECT_EQ(0, a.getSegmentOffset(0));
EXPECT_EQ(0, a.getSegmentIndex(0));
EXPECT_EQ(a.getSize(), a.getSegmentLength(0));
EXPECT_EQ(false, a.isRecursiveAllDone());
a.setSegRecursiveDone(0);
EXPECT_EQ(true, a.isRecursiveAllDone());
a.getSegments()[100] = 1;
a.getSegments()[200] = 1;
a.getSegments()[1000] = 1;
a.getSegments()[a.getSize() - 1] = 1;
a.segmentsChanged();
EXPECT_EQ(5, a.getNumSegments());
a.getSegments()[a.getSize() - 2] = 1;
a.segmentsChanged();
EXPECT_EQ(6, a.getNumSegments());
}
TEST(SegmentArray, LargeArray) {
// This test is named "Negative", and belongs to the "FactorialTest"
// test case.
NeUseDevice(&gDevProp);
int bigsize = gDevProp.totalGlobalMem/(sizeof(int) * 30);
bigsize <<= 1;
CudppPlanFactory planPool;
SegmentArray<int> big(&planPool, bigsize);
SegmentArray<int> zeroArray(&planPool, 0);
EXPECT_EQ(1, big.getNumSegments());
EXPECT_EQ(big.getSize(), big.getSegmentLength(0));
EXPECT_EQ(0, big.getSegmentOffset(0));
EXPECT_EQ(0, big.getSegmentIndex(0));
EXPECT_EQ(big.getSize(), big.getSegmentLength(0));
EXPECT_EQ(false, big.isRecursiveAllDone());
big.setNewSegmentAt(bigsize/2);
EXPECT_EQ(2, big.getNumSegments());
EXPECT_EQ(bigsize/2, big.getSegmentLength(0));
EXPECT_EQ(bigsize/2, big.getSegmentLength(1));
big.setNewSegmentAt(100);
EXPECT_EQ(3, big.getNumSegments());
EXPECT_EQ(100, big.getSegmentLength(0));
EXPECT_EQ((bigsize/2-100), big.getSegmentLength(1));
EXPECT_EQ((bigsize/2), big.getSegmentLength(2));
}
int delta_compress (MemSize BlockSize, int ExtendedTables, CALLBACK_FUNC *callback, void *auxdata)
{
int errcode = FREEARC_OK;
byte *buf = (byte*) BigAlloc(BlockSize); // Buffer for one block of input data (typically, 8mb long)
if (buf==NULL) return FREEARC_ERRCODE_NOT_ENOUGH_MEMORY; // Error: not enough memory
uint64 offset = 0; // Current offset of buf[] contents relative to file (increased after each input block processd)
Buffer TSkip, TType, TRows; // Buffers for storing info about each table filtered
Buffer ReorderingBuffer; // Buffer used in reorder_table
// Each iteration of this cycle reads, process and encodes one block of data
for (;;)
{
// Read input block
int Size; READ_LEN_OR_EOF (Size, buf, BlockSize);
BYTE *bufend = buf + Size; // End of data in buf[]
BYTE *last_table_end = buf; // End of last table found so far
BYTE *hash[256], *hash1[256];
iterate_var(i,256) hash[i] = hash1[i] = buf-1;
for (byte *ptr=buf+LINE; ptr+MAX_ELEMENT_SIZE*4 < bufend; )
{
if (*(int32*)ptr != *(int32*)(ptr+3)) // a little speed optimization, mainly to skip blocks of all zeroes
{
// Посчитаем количество повторений одинаковых или близких байт на разных дистанциях
BYTE count[MAX_ELEMENT_SIZE]; zeroArray(count);
BYTE *p = ptr; iterate_var(i,LINE)
{
int n = p - hash[*p/16]; // detecting repeated data by 4 higher bits
hash[*p/16] = p;
if (n<=MAX_ELEMENT_SIZE) count[n-1]++;
#if 0
// Detecting repeating data by all 8 bits - useful for tables with longer rows
int n1 = p - hash1[*p];
hash1[*p] = p;
if (n!=n1 && n1<=MAX_ELEMENT_SIZE) count[n1-1]++;
#endif
p++;
}
// Теперь отберём те дистанции, на которых было больше 5 повторений -
// это кандидаты на размер строки таблицы
iterate_var(i, MAX_ELEMENT_SIZE) if (count[i] > 5)
{
int N = i+1;
stat ((fast_checks+=N, verbose>1 && printf ("Fast check %08x (%d*%d)\n", int(ptr-buf+offset), N, count[i])));
BYTE *p = ptr;
for (int j=0; j<N; j++, p++) FAST_CHECK_FOR_DATA_TABLE(N,p);
}
}
ptr += LINE;
continue;
// Сюда мы попадаем после того, как найдена и закодирована таблица.
// Пропустим её содержимое
found: ptr = mymax (ptr+LINE, last_table_end);
}
int main() {
int *arr = zeroArray(5);
int i;
for (i = 0; i < 5; i++) {
printf("%d\n", arr[i]);
}
free(arr);
return 0;
}
/*! \brief FlyPacket::reset Resets the packet to all zeros: data,header and footer.
*/
void FlyPacket::reset()
{
readComplete = false;
writeComplete = false;
invalidCommand = false;
byteArrayPositionWrite = PACKET_BEGINNING;
byteArrayPositionRead = PACKET_BEGINNING;
zeroArray(byteArray,sizeof(byteArray));
}
TEST(SegmentArray, ReverseArray) {
NeUseDevice(&gDevProp);
CudppPlanFactory planPool;
SegmentArray<int> a(&planPool, 20);
SegmentArray<int> zeroArray(&planPool, 0);
EXPECT_EQ(1, a.getNumSegments());
EXPECT_EQ(20, a.getSegmentLength(0));
EXPECT_EQ(0, a.getSegmentOffset(0));
EXPECT_EQ(0, a.getSegmentIndex(0));
EXPECT_EQ(false, a.isRecursiveAllDone());
}
int main(void)
{
int word_len[WORD_MAX_LENGTH];
int overflow;
overflow = 0;
zeroArray(word_len, WORD_MAX_LENGTH);
readWords(word_len, WORD_MAX_LENGTH, &overflow);
draw_horizontal_histogram(word_len, WORD_MAX_LENGTH);
printf("\n -------------------------------- \n\n");
draw_vertical_histogram(word_len, WORD_MAX_LENGTH);
printf("num of overflow: %d\n", overflow);
return 0;
}
TEST(SegmentArray, Initialize) {
// This test is named "Negative", and belongs to the "FactorialTest"
// test case.
NeUseDevice(&gDevProp);
CudppPlanFactory planPool;
SegmentArray<int> a(&planPool, 20);
SegmentArray<int> zeroArray(&planPool, 0);
EXPECT_EQ(1, a.getNumSegments());
EXPECT_EQ(20, a.getSegmentLength(0));
EXPECT_EQ(0, a.getSegmentOffset(0));
EXPECT_EQ(0, a.getSegmentIndex(0));
EXPECT_EQ(false, a.isRecursiveAllDone());
a.setNewSegmentAt(10);
EXPECT_EQ(10, a.getSegmentLength(0));
EXPECT_EQ(10, a.getSegmentLength(1));
a.setNewSegmentAt(5);
EXPECT_EQ(5, a.getSegmentLength(0));
EXPECT_EQ(5, a.getSegmentLength(1));
EXPECT_EQ(10, a.getSegmentLength(2));
}
void txReducedPower::prepSrcArrays(
const QVector< QVector<double> > &srcdata
) {
if ( srcdata.isEmpty() ) {
throw txError("Incorrect source data array!");
}
ma_n.clear(); ma_n.resize(m_numberOfPoints);
ma_Me_brutto.clear(); ma_Me_brutto.resize(m_numberOfPoints);
ma_t0.clear(); ma_t0.resize(m_numberOfPoints);
ma_B0.clear(); ma_B0.resize(m_numberOfPoints);
ma_Ra.clear(); ma_Ra.resize(m_numberOfPoints);
ma_S.clear(); ma_S.resize(m_numberOfPoints);
ma_pk.clear(); ma_pk.resize(m_numberOfPoints);
ma_Gfuel.clear(); ma_Gfuel.resize(m_numberOfPoints);
ma_N_k.clear(); ma_N_k.resize(m_numberOfPoints);
ma_N_fan.clear(); ma_N_fan.resize(m_numberOfPoints);
ma_t_cool.clear(); ma_t_cool.resize(m_numberOfPoints);
ma_t_oil.clear(); ma_t_oil.resize(m_numberOfPoints);
ma_tk.clear(); ma_tk.resize(m_numberOfPoints);
ma_tks.clear(); ma_tks.resize(m_numberOfPoints);
ma_t_fuel.clear(); ma_t_fuel.resize(m_numberOfPoints);
ma_pks.clear(); ma_pks.resize(m_numberOfPoints);
ma_Gair.clear(); ma_Gair.resize(m_numberOfPoints);
for ( ptrdiff_t i=0; i<m_numberOfPoints; i++ ) {
if ( srcdata[i].size() != SRCDATACAPTIONS_REDPOWER.size() ) {
throw txError("Incorrect source data array!");
}
ma_n [i] = srcdata[i][ 1];
ma_Me_brutto[i] = srcdata[i][ 2];
ma_t0 [i] = srcdata[i][ 3];
ma_B0 [i] = srcdata[i][ 4];
ma_Ra [i] = srcdata[i][ 5];
ma_S [i] = srcdata[i][ 6];
ma_pk [i] = srcdata[i][ 7];
ma_Gfuel [i] = srcdata[i][ 8];
ma_N_k [i] = srcdata[i][ 9];
ma_N_fan [i] = srcdata[i][10];
ma_t_cool [i] = srcdata[i][11];
ma_t_oil [i] = srcdata[i][12];
ma_tk [i] = srcdata[i][13];
ma_tks [i] = srcdata[i][14];
ma_t_fuel [i] = srcdata[i][15];
ma_pks [i] = srcdata[i][16];
ma_Gair [i] = srcdata[i][17];
}
if ( zeroArray(ma_n) ||
zeroArray(ma_Me_brutto) ||
zeroArray(ma_t0) ||
zeroArray(ma_B0) ||
zeroArray(ma_Ra) ||
zeroArray(ma_pk) ||
zeroArray(ma_Gfuel) ||
(m_calculationOptions->val_Vh() < 0.0000001) ) {
throw txError("Incorrect source data!\n"
"Please check Vh value and arrays of values "
"n, Me, t0, B0, Ra, pk and Gfuel.");
}
}
void HMM::trainGibbsFromFile(char* inputFile)
{
double **emit_count;
double **trans_count;
double *state_count;
double *sequence_count;
double *init_count;
double *obs_count;
emit_count = createMatrix(_numStates, _numObs);
trans_count = createMatrix(_maxState, _numStates);
state_count = new double[_numStates];
zeroArray(state_count, _numStates);
obs_count = new double[_numObs];
zeroArray(obs_count, _numObs);
sequence_count = new double[_maxState];
zeroArray(sequence_count, _maxState);
init_count = new double[_maxState];
zeroArray(init_count, _maxState);
ifstream trainFile(inputFile);
string line;
int count = 0;
while (getline(trainFile, line))
{ // for every sentence
vector<int> words;
stringstream ss(line);
string buf;
while (ss >> buf)
{
words.push_back(atoi(buf.c_str()));
}
int len = words.size();
count++;
int *stateArray;
stateArray = new int[len];
for (int i = 0; i < len; i++)
{
int origState = (rand() % (_numStates - 1)) + 1;
int obs = words[i];
int prev_sequence = 0;
int r = 0;
stateArray[i] = origState;
while ((r < _order) && (i - 1 - r) >= 0)
{
prev_sequence += stateArray[(i - 1) - r] * int(pow(_numStates, r));
r++;
}
obs_count[obs]++;
state_count[origState]++;
if (i == 0)
init_count[origState]++;
else
{
trans_count[prev_sequence][origState]++;
sequence_count[prev_sequence]++;
}
emit_count[origState][obs]++;
}
int sampleN = rand() % len;
for (int i = 0; i < sampleN; i++)
{
int k = rand() % (len - 1);
int obs = words[k];
int prev_sequence = 0;
int r = 0;
// cout << "Compute prev_seq" << endl;
while ((r < _order) && (k - 1 - r) >= 0)
{
prev_sequence += stateArray[(k - 1) - r] * int(pow(_numStates, r));
r++;
}
// cout << "Done Compute prev_seq" << endl;
int origState = stateArray[k];
int nextState = stateArray[k + 1];
int next_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + origState;
double *dist = new double[_numStates];
double totalp = 0;
for (int state = 0; state < _numStates; state++)
{
int state_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + state;
if (prev_sequence == 0)
dist[state] = _pObservation[state][obs] * initial_probability[state]
* _pTransition[state_sequence][nextState];
else
dist[state] = _pObservation[state][obs] * _pTransition[prev_sequence][state]
* _pTransition[state_sequence][nextState];
totalp += dist[state];
}
renormalize(dist, _numStates);
Distribution d(dist, _numStates);
int sample = d.generate_sample();
delete[] dist;
// cout << "Update params" << endl;
state_count[origState]--;
if (k == 0)
{
init_count[origState]--;
init_count[sample]++;
}
else
{
trans_count[prev_sequence][origState]--;
trans_count[prev_sequence][sample]++;
}
trans_count[next_sequence][nextState]--;
sequence_count[next_sequence]--;
emit_count[origState][obs]--;
stateArray[k] = sample;
next_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + sample;
state_count[sample]++;
trans_count[next_sequence][nextState]++;
sequence_count[next_sequence]++;
emit_count[sample][obs]++;
// cout << "Done Update params" << endl;
}
}//end for every sentence
trainFile.close();
updateHMM(emit_count, trans_count, state_count, sequence_count, init_count, obs_count);
freeMatrix(trans_count, _maxState, _numStates);
freeMatrix(emit_count, _numStates, _numObs);
delete[] state_count;
delete[] obs_count;
delete[] sequence_count;
}
void HMM::trainParallel(vector<string> filesList_)
{
int count = filesList_.size();
int rank = MPI::COMM_WORLD.Get_rank();
int size = MPI::COMM_WORLD.Get_size();
const int root = 0;
int dist = count / size;
int start = rank * dist;
int end = rank * dist + dist;
double **emit_count;
double **trans_count;
double *state_count;
double *sequence_count;
double *init_count;
double *obs_count;
emit_count = createMatrix(_numStates, _numObs);
trans_count = createMatrix(_maxState, _numStates);
state_count = new double[_numStates];
zeroArray(state_count, _numStates);
obs_count = new double[_numObs];
zeroArray(obs_count, _numObs);
sequence_count = new double[_maxState];
zeroArray(sequence_count, _maxState);
init_count = new double[_maxState];
zeroArray(init_count, _maxState);
double **temit_count;
double **ttrans_count;
double *tstate_count;
double *tsequence_count;
double *tinit_count;
double *tobs_count;
temit_count = createMatrix(_numStates, _numObs);
ttrans_count = createMatrix(_maxState, _numStates);
tstate_count = new double[_numStates];
tobs_count = new double[_numObs];
zeroArray(tstate_count, _numStates);
tsequence_count = new double[_maxState];
zeroArray(tsequence_count, _maxState);
tinit_count = new double[_maxState];
zeroArray(tinit_count, _maxState);
for (int i = start; i < end; i++)
{
const char* inputFile = filesList_[i].c_str();
// cout << "opening file "<<files_list[i].c_str()<< " On Process " << rank<<endl;
ifstream trainFile(inputFile);
string line;
while (getline(trainFile, line))
{ // for every sentence
// Read in training sequence
vector<int> words;
stringstream ss(line);
string buf;
while (ss >> buf)
{
words.push_back(atoi(buf.c_str()));
}
int len = words.size();
//COMPUTE FORWARD PROBABILITY
double **forward;
double * scaleArray = new double[len];
forward = createMatrix(len, _maxState);
computeForwardMatrixScaled(words, forward, scaleArray, len);
//printMatrix(forward, len, _maxState);
//COMPUTE_BACKWARD PROBABILITY
double **backward;
backward = createMatrix(len, _maxState);
computeBackwardMatrixScaled(words, backward, scaleArray, len);
//printMatrix(backward, len, _maxState);
//BAUM WELCH COUNTS
for (int i = 0; i < len - 1; i++)
{
int obs = words[i];
int next_obs = words[i + 1];
for (int state_sequence = 0; state_sequence < _maxState; state_sequence++)
{
int state = state_sequence % _numStates;
double gamma = (forward[i][state_sequence] * backward[i][state_sequence]);
if (i == 0)
{
init_count[state_sequence] += gamma;
}
emit_count[state][obs] += gamma;
obs_count[obs] += gamma;
state_count[state] += gamma;
sequence_count[state_sequence] += gamma;
for (int next_state = 0; next_state < _numStates; next_state++)
{
int next_sequence = _numStates * (state_sequence % int(pow(_numStates, _order - 1)))
+ next_state;
double eta = (forward[i][state_sequence] * _pTransition[state_sequence][next_state]
* _pObservation[next_state][next_obs] * backward[i + 1][next_sequence]) / scaleArray[i
+ 1];
trans_count[state_sequence][next_state] += eta;
}
}
}
for (int state_sequence = 0; state_sequence < _maxState; state_sequence++)
{
int obs = words[len - 1];
int state = state_sequence % _numStates;
double gamma = (forward[len - 1][state_sequence] * backward[len - 1][state_sequence]);
emit_count[state][obs] += gamma;
obs_count[obs] += gamma;
state_count[state] += gamma;
// sequence_count[state_sequence] += gamma;
}
delete[] scaleArray;
freeMatrix(forward, len, _maxState);
freeMatrix(backward, len, _maxState);
}//end for every sentence
trainFile.close();
// cout << " Training File Close, Updating Parameters " << inputFile << endl;
} // for every file
//Collect parameters on root
for (int state_sequence = 0; state_sequence < _maxState; state_sequence++)
{
MPI_Reduce(trans_count[state_sequence], ttrans_count[state_sequence], _numStates, MPI_DOUBLE, MPI_SUM, root,
MPI_COMM_WORLD);
}
MPI_Reduce(sequence_count, tsequence_count, _maxState, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
MPI_Reduce(init_count, tinit_count, _maxState, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
for (int state = 0; state < _numStates; state++)
{
MPI_Reduce(emit_count[state], temit_count[state], _numObs, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
}
MPI_Reduce(state_count, tstate_count, _numStates, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
MPI_Reduce(obs_count, tobs_count, _numObs, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
//Send updated parameters too all children
for (int state_sequence = 0; state_sequence < _maxState; state_sequence++)
{
MPI_Bcast(ttrans_count[state_sequence], _numStates, MPI_DOUBLE, root, MPI_COMM_WORLD);
}
MPI_Bcast(tsequence_count, _maxState, MPI_DOUBLE, root, MPI_COMM_WORLD);
MPI_Bcast(tinit_count, _maxState, MPI_DOUBLE, root, MPI_COMM_WORLD);
for (int state = 0; state < _numStates; state++)
{
MPI_Bcast(temit_count[state], _numObs, MPI_DOUBLE, root, MPI_COMM_WORLD);
}
MPI_Bcast(tstate_count, _numStates, MPI_DOUBLE, root, MPI_COMM_WORLD);
MPI_Bcast(tobs_count, _numObs, MPI_DOUBLE, root, MPI_COMM_WORLD);
//cout << "Update Step" << endl;
updateHMM(temit_count, ttrans_count, tstate_count, tsequence_count, tinit_count, tobs_count);
freeMatrix(trans_count, _maxState, _numStates);
freeMatrix(emit_count, _numStates, _numObs);
delete[] state_count;
delete[] sequence_count;
delete[] init_count;
delete[] obs_count;
freeMatrix(temit_count, _numStates, _numObs);
freeMatrix(ttrans_count, _maxState, _numStates);
delete[] tstate_count;
delete[] tsequence_count;
delete[] tinit_count;
delete[] tobs_count;
}
void HMM::trainFromFile(char* inputFile)
{
double **emit_count;
double **trans_count;
double *state_count;
double *sequence_count;
double *init_count;
double *obs_count;
emit_count = createMatrix(_numStates, _numObs);
trans_count = createMatrix(_maxState, _numStates);
state_count = new double[_numStates];
zeroArray(state_count, _numStates);
obs_count = new double[_numObs];
zeroArray(obs_count, _numObs);
sequence_count = new double[_maxState];
zeroArray(sequence_count, _maxState);
init_count = new double[_maxState];
zeroArray(init_count, _maxState);
ifstream trainFile(inputFile);
string line;
int count = 0;
while (getline(trainFile, line))
{ // for every sentence
vector<int> words;
stringstream ss(line);
string buf;
while (ss >> buf)
{
words.push_back(atoi(buf.c_str()));
}
int len = words.size();
count++;
//COMPUTE FORWARD PROBABILITY
double **forward;
double * scaleArray = new double[len];
forward = createMatrix(len, _maxState);
//double forward_prob = forwardAlgorithmScaled(words);
//computeForwardMatrix(words, forward, len);
//printMatrix(forward, len, _maxState);
computeForwardMatrixScaled(words, forward, scaleArray, len);
//printMatrix(forward, len, _maxState);
//COMPUTE_BACKWARD PROBABILITY
double **backward;
backward = createMatrix(len, _maxState);
//computeBackwardMatrix(words, backward, len);
//printMatrix(backward, len, _maxState);
computeBackwardMatrixScaled(words, backward, scaleArray, len);
//printMatrix(backward, len, _maxState);
//BAUM WELCH COUNTS
for (int i = 0; i < len - 1; i++)
{
int obs = words[i];
int next_obs = words[i + 1];
for (int state_sequence = 0; state_sequence < _maxState; state_sequence++)
{
int state = state_sequence % _numStates;
//cout << "for state" << state<<" "<<forward[i][state_sequence]<<"*"<<backward[i][state_sequence]<<endl;
double gamma = (forward[i][state_sequence] * backward[i][state_sequence]);
// if (gamma != gamma)
// {
// cout << "gammma problem" << endl;
// printMatrix(forward, len, _maxState);
// printMatrix(backward, len, _maxState);
// printArray(scaleArray, len);
//
// return;
// }
if (i == 0)
{
_pTransition[0][state_sequence] += gamma;
}
emit_count[state][obs] += gamma;
obs_count[obs] += gamma;
state_count[state] += gamma;
sequence_count[state_sequence] += gamma;
for (int next_state = 0; next_state < _numStates; next_state++)
{
int next_sequence = _numStates * (state_sequence % int(pow(_numStates, _order - 1))) + next_state;
double eta = (forward[i][state_sequence] * _pTransition[state_sequence][next_state]
* _pObservation[next_state][next_obs] * backward[i + 1][next_sequence]) / scaleArray[i + 1];
trans_count[state_sequence][next_state] += eta;
}
}
}
for (int state_sequence = 0; state_sequence < _maxState; state_sequence++)
{
int obs = words[len - 1];
int state = state_sequence % _numStates;
double gamma = (forward[len - 1][state_sequence] * backward[len - 1][state_sequence]);
emit_count[state][obs] += gamma;
obs_count[obs] += gamma;
state_count[state] += gamma;
}
delete[] scaleArray;
freeMatrix(forward, len, _maxState);
freeMatrix(backward, len, _maxState);
}//end for every sentence
trainFile.close();
updateHMM(emit_count, trans_count, state_count, sequence_count, init_count, obs_count);
freeMatrix(trans_count, _maxState, _numStates);
freeMatrix(emit_count, _numStates, _numObs);
delete[] state_count;
delete[] obs_count;
delete[] sequence_count;
}
void HMM::trainGibbsParallel(vector<string> files_list)
{
int count = files_list.size();
int rank = MPI::COMM_WORLD.Get_rank();
int size = MPI::COMM_WORLD.Get_size();
const int root = 0;
int dist = count / size;
int start = rank * dist;
int end = rank * dist + dist;
double **emit_count;
double **trans_count;
double *state_count;
double *sequence_count;
double *init_count;
double *obs_count;
emit_count = createMatrix(_numStates, _numObs);
trans_count = createMatrix(_maxState, _numStates);
state_count = new double[_numStates];
zeroArray(state_count, _numStates);
obs_count = new double[_numObs];
zeroArray(obs_count, _numObs);
sequence_count = new double[_maxState];
zeroArray(sequence_count, _maxState);
init_count = new double[_maxState];
zeroArray(init_count, _maxState);
double **temit_count;
double **ttrans_count;
double *tstate_count;
double *tsequence_count;
double *tinit_count;
double *tobs_count;
temit_count = createMatrix(_numStates, _numObs);
ttrans_count = createMatrix(_maxState, _numStates);
tstate_count = new double[_numStates];
zeroArray(tstate_count, _numStates);
tobs_count = new double[_numObs];
zeroArray(tobs_count, _numObs);
tsequence_count = new double[_maxState];
zeroArray(tsequence_count, _maxState);
tinit_count = new double[_maxState];
zeroArray(tinit_count, _maxState);
for (int i = start; i < end; i++)
{
const char* inputFile = files_list[i].c_str();
ifstream trainFile(inputFile);
string line;
while (getline(trainFile, line))
{ // for every sentence
// Read in training sequence
vector<int> words;
stringstream ss(line);
string buf;
while (ss >> buf)
{
words.push_back(atoi(buf.c_str()));
}
int len = words.size();
count++;
int *stateArray;
stateArray = new int[len];
for (int i = 0; i < len; i++)
{
int origState = (rand() % (_numStates - 1)) + 1;
int obs = words[i];
int prev_sequence = 0;
int r = 0;
stateArray[i] = origState;
if (i == 0)
init_count[origState]++;
else
{
while ((r < _order) && (i - 1 - r) >= 0)
{
prev_sequence += stateArray[(i - 1) - r] * int(pow(_numStates, r));
r++;
}
trans_count[prev_sequence][origState]++;
sequence_count[prev_sequence]++;
}
obs_count[obs]++;
state_count[origState]++;
emit_count[origState][obs]++;
}
int sampleN = rand() % len;
for (int i = 0; i < sampleN; i++)
{
int k = rand() % (len - 1);
int obs = words[k];
int prev_sequence = 0;
int r = 0;
// cout << "Compute seq " <<endl;
while ((r < _order) && (k - 1 - r) >= 0)
{
prev_sequence += stateArray[(k - 1) - r] * int(pow(_numStates, r));
r++;
}
// cout << "Done Compute seq " <<endl;
int origState = stateArray[k];
int nextState = stateArray[k + 1];
int next_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + nextState;
double *dist = new double[_numStates];
double totalp = 0;
for (int state = 0; state < _numStates; state++)
{
int state_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + state;
if (prev_sequence == 0)
dist[state] = _pObservation[state][obs] * initial_probability[state]
* _pTransition[state_sequence][nextState];
else
dist[state] = _pObservation[state][obs] * _pTransition[prev_sequence][state]
* _pTransition[state_sequence][nextState];
totalp += dist[state];
}
renormalize(dist, _numStates);
Distribution d(dist, _numStates);
int sample = d.generate_sample();
delete[] dist;
if (k == 0)
{
init_count[origState]--;
init_count[sample]++;
}
else
{
trans_count[prev_sequence][origState]--;
trans_count[prev_sequence][sample]++;
}
state_count[origState]--;
// trans_count[next_sequence][nextState]--;
// sequence_count[next_sequence]--;
emit_count[origState][obs]--;
stateArray[k] = sample;
// next_sequence = _numStates*(prev_sequence % int(pow(_numStates, _order-1))) + sample;
state_count[sample]++;
// trans_count[next_sequence][nextState]++;
// sequence_count[next_sequence]++;
emit_count[sample][obs]++;
// cout << "Done Update parameters" << endl;
}
}//end for every sentence
trainFile.close();
} // for every file
//Collect parameters on root
for (int state_sequence = 0; state_sequence < _maxState; state_sequence++)
{
MPI_Reduce(trans_count[state_sequence], ttrans_count[state_sequence], _numStates, MPI_DOUBLE, MPI_SUM, root,
MPI_COMM_WORLD);
}
MPI_Reduce(sequence_count, tsequence_count, _maxState, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
MPI_Reduce(init_count, tinit_count, _maxState, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
for (int state = 0; state < _numStates; state++)
{
MPI_Reduce(emit_count[state], temit_count[state], _numObs, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
}
MPI_Reduce(state_count, tstate_count, _numStates, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
MPI_Reduce(obs_count, tobs_count, _numObs, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
//Send updated parameters too all children
for (int state_sequence = 0; state_sequence < _maxState; state_sequence++)
{
MPI_Bcast(ttrans_count[state_sequence], _numStates, MPI_DOUBLE, root, MPI_COMM_WORLD);
}
MPI_Bcast(tsequence_count, _maxState, MPI_DOUBLE, root, MPI_COMM_WORLD);
MPI_Bcast(tinit_count, _maxState, MPI_DOUBLE, root, MPI_COMM_WORLD);
for (int state = 0; state < _numStates; state++)
{
MPI_Bcast(temit_count[state], _numObs, MPI_DOUBLE, root, MPI_COMM_WORLD);
}
MPI_Bcast(tstate_count, _numStates, MPI_DOUBLE, root, MPI_COMM_WORLD);
MPI_Bcast(tobs_count, _numObs, MPI_DOUBLE, root, MPI_COMM_WORLD);
//cout << "Update Step" << endl;
updateHMM(temit_count, ttrans_count, tstate_count, tsequence_count, tinit_count, tobs_count);
freeMatrix(trans_count, _maxState, _numStates);
freeMatrix(emit_count, _numStates, _numObs);
delete[] state_count;
delete[] sequence_count;
delete[] init_count;
delete[] obs_count;
freeMatrix(temit_count, _numStates, _numObs);
freeMatrix(ttrans_count, _maxState, _numStates);
delete[] tstate_count;
delete[] tsequence_count;
delete[] tinit_count;
delete[] tobs_count;
}
int main(int argc, char* argv[]) {
const int SIZE = 1 << 8;
const int NPOT = SIZE - 3;
int a[SIZE], b[SIZE], c[SIZE];
float ms_time = 0.0f;
// Scan tests
printf("\n");
printf("****************\n");
printf("** SCAN TESTS **\n");
printf("****************\n");
genArray(SIZE - 1, a, 50); // Leave a 0 at the end to test that edge case
printArray(SIZE, a, true);
zeroArray(SIZE, b);
printDesc("cpu scan, power-of-two");
ms_time = StreamCompaction::CPU::scan(SIZE, b, a);
printf("CPU execution time for scan: %.5fms\n", ms_time);
printArray(SIZE, b, true);
zeroArray(SIZE, c);
printDesc("cpu scan, non-power-of-two");
ms_time = StreamCompaction::CPU::scan(NPOT, c, a);
printf("CPU execution time for scan: %.5fms\n", ms_time);
printArray(NPOT, b, true);
printCmpResult(NPOT, b, c);
zeroArray(SIZE, c);
printDesc("naive scan, power-of-two");
StreamCompaction::Naive::scan(SIZE, c, a);
//printArray(SIZE, c, true);
printCmpResult(SIZE, b, c);
zeroArray(SIZE, c);
printDesc("naive scan, non-power-of-two");
StreamCompaction::Naive::scan(NPOT, c, a);
//printArray(SIZE, c, true);
printCmpResult(NPOT, b, c);
zeroArray(SIZE, c);
printDesc("work-efficient scan, power-of-two");
ms_time = StreamCompaction::Efficient::scan(SIZE, c, a);
printf("CUDA execution time for work efficient scan: %.5fms\n", ms_time);
//printArray(SIZE, c, true);
printCmpResult(SIZE, b, c);
zeroArray(SIZE, c);
printDesc("work-efficient scan, non-power-of-two");
ms_time = StreamCompaction::Efficient::scan(NPOT, c, a);
printf("CUDA execution time for work efficient scan: %.5fms\n", ms_time);
//printArray(NPOT, c, true);
printCmpResult(NPOT, b, c);
zeroArray(SIZE, c);
printDesc("thrust scan, power-of-two");
StreamCompaction::Thrust::scan(SIZE, c, a);
//printArray(SIZE, c, true);
printCmpResult(SIZE, b, c);
zeroArray(SIZE, c);
printDesc("thrust scan, non-power-of-two");
StreamCompaction::Thrust::scan(NPOT, c, a);
//printArray(NPOT, c, true);
printCmpResult(NPOT, b, c);
printf("\n");
printf("*****************************\n");
printf("** STREAM COMPACTION TESTS **\n");
printf("*****************************\n");
// Compaction tests
genArray(SIZE - 1, a, 4); // Leave a 0 at the end to test that edge case
printArray(SIZE, a, true);
int count, expectedCount, expectedNPOT;
zeroArray(SIZE, b);
printDesc("cpu compact without scan, power-of-two");
count = StreamCompaction::CPU::compactWithoutScan(SIZE, b, a);
expectedCount = count;
printArray(count, b, true);
printCmpLenResult(count, expectedCount, b, b);
zeroArray(SIZE, c);
printDesc("cpu compact without scan, non-power-of-two");
count = StreamCompaction::CPU::compactWithoutScan(NPOT, c, a);
expectedNPOT = count;
printArray(count, c, true);
printCmpLenResult(count, expectedNPOT, b, c);
zeroArray(SIZE, c);
printDesc("cpu compact with scan");
count = StreamCompaction::CPU::compactWithScan(SIZE, c, a);
printArray(count, c, true);
printCmpLenResult(count, expectedCount, b, c);
zeroArray(SIZE, c);
printDesc("work-efficient compact, power-of-two");
count = StreamCompaction::Efficient::compact(SIZE, c, a);
//printArray(count, c, true);
printCmpLenResult(count, expectedCount, b, c);
zeroArray(SIZE, c);
printDesc("work-efficient compact, non-power-of-two");
count = StreamCompaction::Efficient::compact(NPOT, c, a);
//printArray(count, c, true);
printCmpLenResult(count, expectedNPOT, b, c);
}
int main(int argc, char* argv[]) {
const int SIZE = 1 << 10;
const int NPOT = SIZE - 3;
int a[SIZE], b[SIZE], c[SIZE];
// Scan tests
printf("\n");
printf("****************\n");
printf("** SCAN TESTS **\n");
printf("****************\n");
genArray(SIZE - 1, a, 50); // Leave a 0 at the end to test that edge case
a[SIZE - 1] = 0;
printArray(SIZE, a, true);
zeroArray(SIZE, b);
printDesc("cpu scan, power-of-two");
StreamCompaction::CPU::scan(SIZE, b, a);
//printArray(SIZE, b, true);
zeroArray(SIZE, c);
printDesc("cpu scan, non-power-of-two");
StreamCompaction::CPU::scan(NPOT, c, a);
//printArray(NPOT, b, true);
printCmpResult(NPOT, b, c);
zeroArray(SIZE, c);
printDesc("naive scan, power-of-two");
StreamCompaction::Naive::scan(SIZE, c, a);
//printArray(SIZE, c, true);
printCmpResult(SIZE, b, c);
zeroArray(SIZE, c);
printDesc("naive scan, non-power-of-two");
StreamCompaction::Naive::scan(NPOT, c, a);
//printArray(NPOT, c, true);
printCmpResult(NPOT, b, c);
zeroArray(SIZE, c);
printDesc("work-efficient scan, power-of-two");
StreamCompaction::Efficient::scan(SIZE, c, a);
//printArray(SIZE, c, true);
printCmpResult(SIZE, b, c);
zeroArray(SIZE, c);
printDesc("work-efficient scan, non-power-of-two");
StreamCompaction::Efficient::scan(NPOT, c, a);
//printArray(NPOT, c, true);
printCmpResult(NPOT, b, c);
zeroArray(SIZE, c);
printDesc("thrust scan, power-of-two");
StreamCompaction::Thrust::scan(SIZE, c, a);
//printArray(SIZE, c, true);
printCmpResult(SIZE, b, c);
zeroArray(SIZE, c);
printDesc("thrust scan, non-power-of-two");
StreamCompaction::Thrust::scan(NPOT, c, a);
//printArray(NPOT, c, true);
printCmpResult(NPOT, b, c);
//*
printf("\n");
printf("*****************************\n");
printf("** STREAM COMPACTION TESTS **\n");
printf("*****************************\n");
// Compaction tests
genArray(SIZE - 1, a, 4); // Leave a 0 at the end to test that edge case
printArray(SIZE, a, true);
a[SIZE - 1] = 0;
int count, expectedCount, expectedNPOT;
zeroArray(SIZE, b);
printDesc("cpu compact without scan, power-of-two");
count = StreamCompaction::CPU::compactWithoutScan(SIZE, b, a);
expectedCount = count;
printArray(count, b, true);
printCmpLenResult(count, expectedCount, b, b);
zeroArray(SIZE, c);
printDesc("cpu compact without scan, non-power-of-two");
count = StreamCompaction::CPU::compactWithoutScan(NPOT, c, a);
expectedNPOT = count;
printArray(count, c, true);
printCmpLenResult(count, expectedNPOT, b, c);
zeroArray(SIZE, c);
printDesc("cpu compact with scan");
count = StreamCompaction::CPU::compactWithScan(SIZE, c, a);
printArray(count, c, true);
printCmpLenResult(count, expectedCount, b, c);
zeroArray(SIZE, c);
printDesc("work-efficient compact, power-of-two");
printArray(SIZE, a, true);//delete
count = StreamCompaction::Efficient::compact(SIZE, c, a);
printArray(count, c, true);
printCmpLenResult(count, expectedCount, b, c);
zeroArray(SIZE, c);
printDesc("work-efficient compact, non-power-of-two");
printArray(SIZE, a, true);//delete
count = StreamCompaction::Efficient::compact(NPOT, c, a);
printArray(count, c, true);
printCmpLenResult(count, expectedNPOT, b, c);
printf("\n");
printf("*****************************\n");
printf("** Radix Sort **\n");
printf("*****************************\n");
a[0] = 4;
a[1] = 7;
a[2] = 2;
a[3] = 6;
a[4] = 3;
a[5] = 5;
a[6] = 1;
a[7] = 0;
zeroArray(8, c);
printDesc("Radix Sort, power-of-two");
printArray(8, a, true);//delete
StreamCompaction::RadixSort::sort(8, c, a);
printArray(8, c, true);
//printCmpResult(SIZE, b, c);*/
std::cin.get();
}
