TEST(SegmentArray, LargeArray) { // This test is named "Negative", and belongs to the "FactorialTest" // test case. NeUseDevice(&gDevProp); int bigsize = gDevProp.totalGlobalMem/(sizeof(int) * 20); CudppPlanFactory planPool; SegmentArray<int> a(&planPool, bigsize); SegmentArray<int> zeroArray(&planPool, 0); EXPECT_EQ(1, a.getNumSegments()); EXPECT_EQ(a.getSize(), a.getSegmentLength(0)); EXPECT_EQ(0, a.getSegmentOffset(0)); EXPECT_EQ(0, a.getSegmentIndex(0)); EXPECT_EQ(a.getSize(), a.getSegmentLength(0)); EXPECT_EQ(false, a.isRecursiveAllDone()); a.setSegRecursiveDone(0); EXPECT_EQ(true, a.isRecursiveAllDone()); a.getSegments()[100] = 1; a.getSegments()[200] = 1; a.getSegments()[1000] = 1; a.getSegments()[a.getSize() - 1] = 1; a.segmentsChanged(); EXPECT_EQ(5, a.getNumSegments()); a.getSegments()[a.getSize() - 2] = 1; a.segmentsChanged(); EXPECT_EQ(6, a.getNumSegments()); }
TEST(SegmentArray, LargeArray) { // This test is named "Negative", and belongs to the "FactorialTest" // test case. NeUseDevice(&gDevProp); int bigsize = gDevProp.totalGlobalMem/(sizeof(int) * 30); bigsize <<= 1; CudppPlanFactory planPool; SegmentArray<int> big(&planPool, bigsize); SegmentArray<int> zeroArray(&planPool, 0); EXPECT_EQ(1, big.getNumSegments()); EXPECT_EQ(big.getSize(), big.getSegmentLength(0)); EXPECT_EQ(0, big.getSegmentOffset(0)); EXPECT_EQ(0, big.getSegmentIndex(0)); EXPECT_EQ(big.getSize(), big.getSegmentLength(0)); EXPECT_EQ(false, big.isRecursiveAllDone()); big.setNewSegmentAt(bigsize/2); EXPECT_EQ(2, big.getNumSegments()); EXPECT_EQ(bigsize/2, big.getSegmentLength(0)); EXPECT_EQ(bigsize/2, big.getSegmentLength(1)); big.setNewSegmentAt(100); EXPECT_EQ(3, big.getNumSegments()); EXPECT_EQ(100, big.getSegmentLength(0)); EXPECT_EQ((bigsize/2-100), big.getSegmentLength(1)); EXPECT_EQ((bigsize/2), big.getSegmentLength(2)); }
int delta_compress (MemSize BlockSize, int ExtendedTables, CALLBACK_FUNC *callback, void *auxdata) { int errcode = FREEARC_OK; byte *buf = (byte*) BigAlloc(BlockSize); // Buffer for one block of input data (typically, 8mb long) if (buf==NULL) return FREEARC_ERRCODE_NOT_ENOUGH_MEMORY; // Error: not enough memory uint64 offset = 0; // Current offset of buf[] contents relative to file (increased after each input block processd) Buffer TSkip, TType, TRows; // Buffers for storing info about each table filtered Buffer ReorderingBuffer; // Buffer used in reorder_table // Each iteration of this cycle reads, process and encodes one block of data for (;;) { // Read input block int Size; READ_LEN_OR_EOF (Size, buf, BlockSize); BYTE *bufend = buf + Size; // End of data in buf[] BYTE *last_table_end = buf; // End of last table found so far BYTE *hash[256], *hash1[256]; iterate_var(i,256) hash[i] = hash1[i] = buf-1; for (byte *ptr=buf+LINE; ptr+MAX_ELEMENT_SIZE*4 < bufend; ) { if (*(int32*)ptr != *(int32*)(ptr+3)) // a little speed optimization, mainly to skip blocks of all zeroes { // Посчитаем количество повторений одинаковых или близких байт на разных дистанциях BYTE count[MAX_ELEMENT_SIZE]; zeroArray(count); BYTE *p = ptr; iterate_var(i,LINE) { int n = p - hash[*p/16]; // detecting repeated data by 4 higher bits hash[*p/16] = p; if (n<=MAX_ELEMENT_SIZE) count[n-1]++; #if 0 // Detecting repeating data by all 8 bits - useful for tables with longer rows int n1 = p - hash1[*p]; hash1[*p] = p; if (n!=n1 && n1<=MAX_ELEMENT_SIZE) count[n1-1]++; #endif p++; } // Теперь отберём те дистанции, на которых было больше 5 повторений - // это кандидаты на размер строки таблицы iterate_var(i, MAX_ELEMENT_SIZE) if (count[i] > 5) { int N = i+1; stat ((fast_checks+=N, verbose>1 && printf ("Fast check %08x (%d*%d)\n", int(ptr-buf+offset), N, count[i]))); BYTE *p = ptr; for (int j=0; j<N; j++, p++) FAST_CHECK_FOR_DATA_TABLE(N,p); } } ptr += LINE; continue; // Сюда мы попадаем после того, как найдена и закодирована таблица. // Пропустим её содержимое found: ptr = mymax (ptr+LINE, last_table_end); }
int main() { int *arr = zeroArray(5); int i; for (i = 0; i < 5; i++) { printf("%d\n", arr[i]); } free(arr); return 0; }
/*! \brief FlyPacket::reset Resets the packet to all zeros: data,header and footer. */ void FlyPacket::reset() { readComplete = false; writeComplete = false; invalidCommand = false; byteArrayPositionWrite = PACKET_BEGINNING; byteArrayPositionRead = PACKET_BEGINNING; zeroArray(byteArray,sizeof(byteArray)); }
TEST(SegmentArray, ReverseArray) { NeUseDevice(&gDevProp); CudppPlanFactory planPool; SegmentArray<int> a(&planPool, 20); SegmentArray<int> zeroArray(&planPool, 0); EXPECT_EQ(1, a.getNumSegments()); EXPECT_EQ(20, a.getSegmentLength(0)); EXPECT_EQ(0, a.getSegmentOffset(0)); EXPECT_EQ(0, a.getSegmentIndex(0)); EXPECT_EQ(false, a.isRecursiveAllDone()); }
int main(void) { int word_len[WORD_MAX_LENGTH]; int overflow; overflow = 0; zeroArray(word_len, WORD_MAX_LENGTH); readWords(word_len, WORD_MAX_LENGTH, &overflow); draw_horizontal_histogram(word_len, WORD_MAX_LENGTH); printf("\n -------------------------------- \n\n"); draw_vertical_histogram(word_len, WORD_MAX_LENGTH); printf("num of overflow: %d\n", overflow); return 0; }
TEST(SegmentArray, Initialize) { // This test is named "Negative", and belongs to the "FactorialTest" // test case. NeUseDevice(&gDevProp); CudppPlanFactory planPool; SegmentArray<int> a(&planPool, 20); SegmentArray<int> zeroArray(&planPool, 0); EXPECT_EQ(1, a.getNumSegments()); EXPECT_EQ(20, a.getSegmentLength(0)); EXPECT_EQ(0, a.getSegmentOffset(0)); EXPECT_EQ(0, a.getSegmentIndex(0)); EXPECT_EQ(false, a.isRecursiveAllDone()); a.setNewSegmentAt(10); EXPECT_EQ(10, a.getSegmentLength(0)); EXPECT_EQ(10, a.getSegmentLength(1)); a.setNewSegmentAt(5); EXPECT_EQ(5, a.getSegmentLength(0)); EXPECT_EQ(5, a.getSegmentLength(1)); EXPECT_EQ(10, a.getSegmentLength(2)); }
void txReducedPower::prepSrcArrays( const QVector< QVector<double> > &srcdata ) { if ( srcdata.isEmpty() ) { throw txError("Incorrect source data array!"); } ma_n.clear(); ma_n.resize(m_numberOfPoints); ma_Me_brutto.clear(); ma_Me_brutto.resize(m_numberOfPoints); ma_t0.clear(); ma_t0.resize(m_numberOfPoints); ma_B0.clear(); ma_B0.resize(m_numberOfPoints); ma_Ra.clear(); ma_Ra.resize(m_numberOfPoints); ma_S.clear(); ma_S.resize(m_numberOfPoints); ma_pk.clear(); ma_pk.resize(m_numberOfPoints); ma_Gfuel.clear(); ma_Gfuel.resize(m_numberOfPoints); ma_N_k.clear(); ma_N_k.resize(m_numberOfPoints); ma_N_fan.clear(); ma_N_fan.resize(m_numberOfPoints); ma_t_cool.clear(); ma_t_cool.resize(m_numberOfPoints); ma_t_oil.clear(); ma_t_oil.resize(m_numberOfPoints); ma_tk.clear(); ma_tk.resize(m_numberOfPoints); ma_tks.clear(); ma_tks.resize(m_numberOfPoints); ma_t_fuel.clear(); ma_t_fuel.resize(m_numberOfPoints); ma_pks.clear(); ma_pks.resize(m_numberOfPoints); ma_Gair.clear(); ma_Gair.resize(m_numberOfPoints); for ( ptrdiff_t i=0; i<m_numberOfPoints; i++ ) { if ( srcdata[i].size() != SRCDATACAPTIONS_REDPOWER.size() ) { throw txError("Incorrect source data array!"); } ma_n [i] = srcdata[i][ 1]; ma_Me_brutto[i] = srcdata[i][ 2]; ma_t0 [i] = srcdata[i][ 3]; ma_B0 [i] = srcdata[i][ 4]; ma_Ra [i] = srcdata[i][ 5]; ma_S [i] = srcdata[i][ 6]; ma_pk [i] = srcdata[i][ 7]; ma_Gfuel [i] = srcdata[i][ 8]; ma_N_k [i] = srcdata[i][ 9]; ma_N_fan [i] = srcdata[i][10]; ma_t_cool [i] = srcdata[i][11]; ma_t_oil [i] = srcdata[i][12]; ma_tk [i] = srcdata[i][13]; ma_tks [i] = srcdata[i][14]; ma_t_fuel [i] = srcdata[i][15]; ma_pks [i] = srcdata[i][16]; ma_Gair [i] = srcdata[i][17]; } if ( zeroArray(ma_n) || zeroArray(ma_Me_brutto) || zeroArray(ma_t0) || zeroArray(ma_B0) || zeroArray(ma_Ra) || zeroArray(ma_pk) || zeroArray(ma_Gfuel) || (m_calculationOptions->val_Vh() < 0.0000001) ) { throw txError("Incorrect source data!\n" "Please check Vh value and arrays of values " "n, Me, t0, B0, Ra, pk and Gfuel."); } }
void HMM::trainGibbsFromFile(char* inputFile) { double **emit_count; double **trans_count; double *state_count; double *sequence_count; double *init_count; double *obs_count; emit_count = createMatrix(_numStates, _numObs); trans_count = createMatrix(_maxState, _numStates); state_count = new double[_numStates]; zeroArray(state_count, _numStates); obs_count = new double[_numObs]; zeroArray(obs_count, _numObs); sequence_count = new double[_maxState]; zeroArray(sequence_count, _maxState); init_count = new double[_maxState]; zeroArray(init_count, _maxState); ifstream trainFile(inputFile); string line; int count = 0; while (getline(trainFile, line)) { // for every sentence vector<int> words; stringstream ss(line); string buf; while (ss >> buf) { words.push_back(atoi(buf.c_str())); } int len = words.size(); count++; int *stateArray; stateArray = new int[len]; for (int i = 0; i < len; i++) { int origState = (rand() % (_numStates - 1)) + 1; int obs = words[i]; int prev_sequence = 0; int r = 0; stateArray[i] = origState; while ((r < _order) && (i - 1 - r) >= 0) { prev_sequence += stateArray[(i - 1) - r] * int(pow(_numStates, r)); r++; } obs_count[obs]++; state_count[origState]++; if (i == 0) init_count[origState]++; else { trans_count[prev_sequence][origState]++; sequence_count[prev_sequence]++; } emit_count[origState][obs]++; } int sampleN = rand() % len; for (int i = 0; i < sampleN; i++) { int k = rand() % (len - 1); int obs = words[k]; int prev_sequence = 0; int r = 0; // cout << "Compute prev_seq" << endl; while ((r < _order) && (k - 1 - r) >= 0) { prev_sequence += stateArray[(k - 1) - r] * int(pow(_numStates, r)); r++; } // cout << "Done Compute prev_seq" << endl; int origState = stateArray[k]; int nextState = stateArray[k + 1]; int next_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + origState; double *dist = new double[_numStates]; double totalp = 0; for (int state = 0; state < _numStates; state++) { int state_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + state; if (prev_sequence == 0) dist[state] = _pObservation[state][obs] * initial_probability[state] * _pTransition[state_sequence][nextState]; else dist[state] = _pObservation[state][obs] * _pTransition[prev_sequence][state] * _pTransition[state_sequence][nextState]; totalp += dist[state]; } renormalize(dist, _numStates); Distribution d(dist, _numStates); int sample = d.generate_sample(); delete[] dist; // cout << "Update params" << endl; state_count[origState]--; if (k == 0) { init_count[origState]--; init_count[sample]++; } else { trans_count[prev_sequence][origState]--; trans_count[prev_sequence][sample]++; } trans_count[next_sequence][nextState]--; sequence_count[next_sequence]--; emit_count[origState][obs]--; stateArray[k] = sample; next_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + sample; state_count[sample]++; trans_count[next_sequence][nextState]++; sequence_count[next_sequence]++; emit_count[sample][obs]++; // cout << "Done Update params" << endl; } }//end for every sentence trainFile.close(); updateHMM(emit_count, trans_count, state_count, sequence_count, init_count, obs_count); freeMatrix(trans_count, _maxState, _numStates); freeMatrix(emit_count, _numStates, _numObs); delete[] state_count; delete[] obs_count; delete[] sequence_count; }
void HMM::trainParallel(vector<string> filesList_) { int count = filesList_.size(); int rank = MPI::COMM_WORLD.Get_rank(); int size = MPI::COMM_WORLD.Get_size(); const int root = 0; int dist = count / size; int start = rank * dist; int end = rank * dist + dist; double **emit_count; double **trans_count; double *state_count; double *sequence_count; double *init_count; double *obs_count; emit_count = createMatrix(_numStates, _numObs); trans_count = createMatrix(_maxState, _numStates); state_count = new double[_numStates]; zeroArray(state_count, _numStates); obs_count = new double[_numObs]; zeroArray(obs_count, _numObs); sequence_count = new double[_maxState]; zeroArray(sequence_count, _maxState); init_count = new double[_maxState]; zeroArray(init_count, _maxState); double **temit_count; double **ttrans_count; double *tstate_count; double *tsequence_count; double *tinit_count; double *tobs_count; temit_count = createMatrix(_numStates, _numObs); ttrans_count = createMatrix(_maxState, _numStates); tstate_count = new double[_numStates]; tobs_count = new double[_numObs]; zeroArray(tstate_count, _numStates); tsequence_count = new double[_maxState]; zeroArray(tsequence_count, _maxState); tinit_count = new double[_maxState]; zeroArray(tinit_count, _maxState); for (int i = start; i < end; i++) { const char* inputFile = filesList_[i].c_str(); // cout << "opening file "<<files_list[i].c_str()<< " On Process " << rank<<endl; ifstream trainFile(inputFile); string line; while (getline(trainFile, line)) { // for every sentence // Read in training sequence vector<int> words; stringstream ss(line); string buf; while (ss >> buf) { words.push_back(atoi(buf.c_str())); } int len = words.size(); //COMPUTE FORWARD PROBABILITY double **forward; double * scaleArray = new double[len]; forward = createMatrix(len, _maxState); computeForwardMatrixScaled(words, forward, scaleArray, len); //printMatrix(forward, len, _maxState); //COMPUTE_BACKWARD PROBABILITY double **backward; backward = createMatrix(len, _maxState); computeBackwardMatrixScaled(words, backward, scaleArray, len); //printMatrix(backward, len, _maxState); //BAUM WELCH COUNTS for (int i = 0; i < len - 1; i++) { int obs = words[i]; int next_obs = words[i + 1]; for (int state_sequence = 0; state_sequence < _maxState; state_sequence++) { int state = state_sequence % _numStates; double gamma = (forward[i][state_sequence] * backward[i][state_sequence]); if (i == 0) { init_count[state_sequence] += gamma; } emit_count[state][obs] += gamma; obs_count[obs] += gamma; state_count[state] += gamma; sequence_count[state_sequence] += gamma; for (int next_state = 0; next_state < _numStates; next_state++) { int next_sequence = _numStates * (state_sequence % int(pow(_numStates, _order - 1))) + next_state; double eta = (forward[i][state_sequence] * _pTransition[state_sequence][next_state] * _pObservation[next_state][next_obs] * backward[i + 1][next_sequence]) / scaleArray[i + 1]; trans_count[state_sequence][next_state] += eta; } } } for (int state_sequence = 0; state_sequence < _maxState; state_sequence++) { int obs = words[len - 1]; int state = state_sequence % _numStates; double gamma = (forward[len - 1][state_sequence] * backward[len - 1][state_sequence]); emit_count[state][obs] += gamma; obs_count[obs] += gamma; state_count[state] += gamma; // sequence_count[state_sequence] += gamma; } delete[] scaleArray; freeMatrix(forward, len, _maxState); freeMatrix(backward, len, _maxState); }//end for every sentence trainFile.close(); // cout << " Training File Close, Updating Parameters " << inputFile << endl; } // for every file //Collect parameters on root for (int state_sequence = 0; state_sequence < _maxState; state_sequence++) { MPI_Reduce(trans_count[state_sequence], ttrans_count[state_sequence], _numStates, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); } MPI_Reduce(sequence_count, tsequence_count, _maxState, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); MPI_Reduce(init_count, tinit_count, _maxState, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); for (int state = 0; state < _numStates; state++) { MPI_Reduce(emit_count[state], temit_count[state], _numObs, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); } MPI_Reduce(state_count, tstate_count, _numStates, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); MPI_Reduce(obs_count, tobs_count, _numObs, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); //Send updated parameters too all children for (int state_sequence = 0; state_sequence < _maxState; state_sequence++) { MPI_Bcast(ttrans_count[state_sequence], _numStates, MPI_DOUBLE, root, MPI_COMM_WORLD); } MPI_Bcast(tsequence_count, _maxState, MPI_DOUBLE, root, MPI_COMM_WORLD); MPI_Bcast(tinit_count, _maxState, MPI_DOUBLE, root, MPI_COMM_WORLD); for (int state = 0; state < _numStates; state++) { MPI_Bcast(temit_count[state], _numObs, MPI_DOUBLE, root, MPI_COMM_WORLD); } MPI_Bcast(tstate_count, _numStates, MPI_DOUBLE, root, MPI_COMM_WORLD); MPI_Bcast(tobs_count, _numObs, MPI_DOUBLE, root, MPI_COMM_WORLD); //cout << "Update Step" << endl; updateHMM(temit_count, ttrans_count, tstate_count, tsequence_count, tinit_count, tobs_count); freeMatrix(trans_count, _maxState, _numStates); freeMatrix(emit_count, _numStates, _numObs); delete[] state_count; delete[] sequence_count; delete[] init_count; delete[] obs_count; freeMatrix(temit_count, _numStates, _numObs); freeMatrix(ttrans_count, _maxState, _numStates); delete[] tstate_count; delete[] tsequence_count; delete[] tinit_count; delete[] tobs_count; }
void HMM::trainFromFile(char* inputFile) { double **emit_count; double **trans_count; double *state_count; double *sequence_count; double *init_count; double *obs_count; emit_count = createMatrix(_numStates, _numObs); trans_count = createMatrix(_maxState, _numStates); state_count = new double[_numStates]; zeroArray(state_count, _numStates); obs_count = new double[_numObs]; zeroArray(obs_count, _numObs); sequence_count = new double[_maxState]; zeroArray(sequence_count, _maxState); init_count = new double[_maxState]; zeroArray(init_count, _maxState); ifstream trainFile(inputFile); string line; int count = 0; while (getline(trainFile, line)) { // for every sentence vector<int> words; stringstream ss(line); string buf; while (ss >> buf) { words.push_back(atoi(buf.c_str())); } int len = words.size(); count++; //COMPUTE FORWARD PROBABILITY double **forward; double * scaleArray = new double[len]; forward = createMatrix(len, _maxState); //double forward_prob = forwardAlgorithmScaled(words); //computeForwardMatrix(words, forward, len); //printMatrix(forward, len, _maxState); computeForwardMatrixScaled(words, forward, scaleArray, len); //printMatrix(forward, len, _maxState); //COMPUTE_BACKWARD PROBABILITY double **backward; backward = createMatrix(len, _maxState); //computeBackwardMatrix(words, backward, len); //printMatrix(backward, len, _maxState); computeBackwardMatrixScaled(words, backward, scaleArray, len); //printMatrix(backward, len, _maxState); //BAUM WELCH COUNTS for (int i = 0; i < len - 1; i++) { int obs = words[i]; int next_obs = words[i + 1]; for (int state_sequence = 0; state_sequence < _maxState; state_sequence++) { int state = state_sequence % _numStates; //cout << "for state" << state<<" "<<forward[i][state_sequence]<<"*"<<backward[i][state_sequence]<<endl; double gamma = (forward[i][state_sequence] * backward[i][state_sequence]); // if (gamma != gamma) // { // cout << "gammma problem" << endl; // printMatrix(forward, len, _maxState); // printMatrix(backward, len, _maxState); // printArray(scaleArray, len); // // return; // } if (i == 0) { _pTransition[0][state_sequence] += gamma; } emit_count[state][obs] += gamma; obs_count[obs] += gamma; state_count[state] += gamma; sequence_count[state_sequence] += gamma; for (int next_state = 0; next_state < _numStates; next_state++) { int next_sequence = _numStates * (state_sequence % int(pow(_numStates, _order - 1))) + next_state; double eta = (forward[i][state_sequence] * _pTransition[state_sequence][next_state] * _pObservation[next_state][next_obs] * backward[i + 1][next_sequence]) / scaleArray[i + 1]; trans_count[state_sequence][next_state] += eta; } } } for (int state_sequence = 0; state_sequence < _maxState; state_sequence++) { int obs = words[len - 1]; int state = state_sequence % _numStates; double gamma = (forward[len - 1][state_sequence] * backward[len - 1][state_sequence]); emit_count[state][obs] += gamma; obs_count[obs] += gamma; state_count[state] += gamma; } delete[] scaleArray; freeMatrix(forward, len, _maxState); freeMatrix(backward, len, _maxState); }//end for every sentence trainFile.close(); updateHMM(emit_count, trans_count, state_count, sequence_count, init_count, obs_count); freeMatrix(trans_count, _maxState, _numStates); freeMatrix(emit_count, _numStates, _numObs); delete[] state_count; delete[] obs_count; delete[] sequence_count; }
void HMM::trainGibbsParallel(vector<string> files_list) { int count = files_list.size(); int rank = MPI::COMM_WORLD.Get_rank(); int size = MPI::COMM_WORLD.Get_size(); const int root = 0; int dist = count / size; int start = rank * dist; int end = rank * dist + dist; double **emit_count; double **trans_count; double *state_count; double *sequence_count; double *init_count; double *obs_count; emit_count = createMatrix(_numStates, _numObs); trans_count = createMatrix(_maxState, _numStates); state_count = new double[_numStates]; zeroArray(state_count, _numStates); obs_count = new double[_numObs]; zeroArray(obs_count, _numObs); sequence_count = new double[_maxState]; zeroArray(sequence_count, _maxState); init_count = new double[_maxState]; zeroArray(init_count, _maxState); double **temit_count; double **ttrans_count; double *tstate_count; double *tsequence_count; double *tinit_count; double *tobs_count; temit_count = createMatrix(_numStates, _numObs); ttrans_count = createMatrix(_maxState, _numStates); tstate_count = new double[_numStates]; zeroArray(tstate_count, _numStates); tobs_count = new double[_numObs]; zeroArray(tobs_count, _numObs); tsequence_count = new double[_maxState]; zeroArray(tsequence_count, _maxState); tinit_count = new double[_maxState]; zeroArray(tinit_count, _maxState); for (int i = start; i < end; i++) { const char* inputFile = files_list[i].c_str(); ifstream trainFile(inputFile); string line; while (getline(trainFile, line)) { // for every sentence // Read in training sequence vector<int> words; stringstream ss(line); string buf; while (ss >> buf) { words.push_back(atoi(buf.c_str())); } int len = words.size(); count++; int *stateArray; stateArray = new int[len]; for (int i = 0; i < len; i++) { int origState = (rand() % (_numStates - 1)) + 1; int obs = words[i]; int prev_sequence = 0; int r = 0; stateArray[i] = origState; if (i == 0) init_count[origState]++; else { while ((r < _order) && (i - 1 - r) >= 0) { prev_sequence += stateArray[(i - 1) - r] * int(pow(_numStates, r)); r++; } trans_count[prev_sequence][origState]++; sequence_count[prev_sequence]++; } obs_count[obs]++; state_count[origState]++; emit_count[origState][obs]++; } int sampleN = rand() % len; for (int i = 0; i < sampleN; i++) { int k = rand() % (len - 1); int obs = words[k]; int prev_sequence = 0; int r = 0; // cout << "Compute seq " <<endl; while ((r < _order) && (k - 1 - r) >= 0) { prev_sequence += stateArray[(k - 1) - r] * int(pow(_numStates, r)); r++; } // cout << "Done Compute seq " <<endl; int origState = stateArray[k]; int nextState = stateArray[k + 1]; int next_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + nextState; double *dist = new double[_numStates]; double totalp = 0; for (int state = 0; state < _numStates; state++) { int state_sequence = _numStates * (prev_sequence % int(pow(_numStates, _order - 1))) + state; if (prev_sequence == 0) dist[state] = _pObservation[state][obs] * initial_probability[state] * _pTransition[state_sequence][nextState]; else dist[state] = _pObservation[state][obs] * _pTransition[prev_sequence][state] * _pTransition[state_sequence][nextState]; totalp += dist[state]; } renormalize(dist, _numStates); Distribution d(dist, _numStates); int sample = d.generate_sample(); delete[] dist; if (k == 0) { init_count[origState]--; init_count[sample]++; } else { trans_count[prev_sequence][origState]--; trans_count[prev_sequence][sample]++; } state_count[origState]--; // trans_count[next_sequence][nextState]--; // sequence_count[next_sequence]--; emit_count[origState][obs]--; stateArray[k] = sample; // next_sequence = _numStates*(prev_sequence % int(pow(_numStates, _order-1))) + sample; state_count[sample]++; // trans_count[next_sequence][nextState]++; // sequence_count[next_sequence]++; emit_count[sample][obs]++; // cout << "Done Update parameters" << endl; } }//end for every sentence trainFile.close(); } // for every file //Collect parameters on root for (int state_sequence = 0; state_sequence < _maxState; state_sequence++) { MPI_Reduce(trans_count[state_sequence], ttrans_count[state_sequence], _numStates, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); } MPI_Reduce(sequence_count, tsequence_count, _maxState, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); MPI_Reduce(init_count, tinit_count, _maxState, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); for (int state = 0; state < _numStates; state++) { MPI_Reduce(emit_count[state], temit_count[state], _numObs, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); } MPI_Reduce(state_count, tstate_count, _numStates, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); MPI_Reduce(obs_count, tobs_count, _numObs, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD); //Send updated parameters too all children for (int state_sequence = 0; state_sequence < _maxState; state_sequence++) { MPI_Bcast(ttrans_count[state_sequence], _numStates, MPI_DOUBLE, root, MPI_COMM_WORLD); } MPI_Bcast(tsequence_count, _maxState, MPI_DOUBLE, root, MPI_COMM_WORLD); MPI_Bcast(tinit_count, _maxState, MPI_DOUBLE, root, MPI_COMM_WORLD); for (int state = 0; state < _numStates; state++) { MPI_Bcast(temit_count[state], _numObs, MPI_DOUBLE, root, MPI_COMM_WORLD); } MPI_Bcast(tstate_count, _numStates, MPI_DOUBLE, root, MPI_COMM_WORLD); MPI_Bcast(tobs_count, _numObs, MPI_DOUBLE, root, MPI_COMM_WORLD); //cout << "Update Step" << endl; updateHMM(temit_count, ttrans_count, tstate_count, tsequence_count, tinit_count, tobs_count); freeMatrix(trans_count, _maxState, _numStates); freeMatrix(emit_count, _numStates, _numObs); delete[] state_count; delete[] sequence_count; delete[] init_count; delete[] obs_count; freeMatrix(temit_count, _numStates, _numObs); freeMatrix(ttrans_count, _maxState, _numStates); delete[] tstate_count; delete[] tsequence_count; delete[] tinit_count; delete[] tobs_count; }
int main(int argc, char* argv[]) { const int SIZE = 1 << 8; const int NPOT = SIZE - 3; int a[SIZE], b[SIZE], c[SIZE]; float ms_time = 0.0f; // Scan tests printf("\n"); printf("****************\n"); printf("** SCAN TESTS **\n"); printf("****************\n"); genArray(SIZE - 1, a, 50); // Leave a 0 at the end to test that edge case printArray(SIZE, a, true); zeroArray(SIZE, b); printDesc("cpu scan, power-of-two"); ms_time = StreamCompaction::CPU::scan(SIZE, b, a); printf("CPU execution time for scan: %.5fms\n", ms_time); printArray(SIZE, b, true); zeroArray(SIZE, c); printDesc("cpu scan, non-power-of-two"); ms_time = StreamCompaction::CPU::scan(NPOT, c, a); printf("CPU execution time for scan: %.5fms\n", ms_time); printArray(NPOT, b, true); printCmpResult(NPOT, b, c); zeroArray(SIZE, c); printDesc("naive scan, power-of-two"); StreamCompaction::Naive::scan(SIZE, c, a); //printArray(SIZE, c, true); printCmpResult(SIZE, b, c); zeroArray(SIZE, c); printDesc("naive scan, non-power-of-two"); StreamCompaction::Naive::scan(NPOT, c, a); //printArray(SIZE, c, true); printCmpResult(NPOT, b, c); zeroArray(SIZE, c); printDesc("work-efficient scan, power-of-two"); ms_time = StreamCompaction::Efficient::scan(SIZE, c, a); printf("CUDA execution time for work efficient scan: %.5fms\n", ms_time); //printArray(SIZE, c, true); printCmpResult(SIZE, b, c); zeroArray(SIZE, c); printDesc("work-efficient scan, non-power-of-two"); ms_time = StreamCompaction::Efficient::scan(NPOT, c, a); printf("CUDA execution time for work efficient scan: %.5fms\n", ms_time); //printArray(NPOT, c, true); printCmpResult(NPOT, b, c); zeroArray(SIZE, c); printDesc("thrust scan, power-of-two"); StreamCompaction::Thrust::scan(SIZE, c, a); //printArray(SIZE, c, true); printCmpResult(SIZE, b, c); zeroArray(SIZE, c); printDesc("thrust scan, non-power-of-two"); StreamCompaction::Thrust::scan(NPOT, c, a); //printArray(NPOT, c, true); printCmpResult(NPOT, b, c); printf("\n"); printf("*****************************\n"); printf("** STREAM COMPACTION TESTS **\n"); printf("*****************************\n"); // Compaction tests genArray(SIZE - 1, a, 4); // Leave a 0 at the end to test that edge case printArray(SIZE, a, true); int count, expectedCount, expectedNPOT; zeroArray(SIZE, b); printDesc("cpu compact without scan, power-of-two"); count = StreamCompaction::CPU::compactWithoutScan(SIZE, b, a); expectedCount = count; printArray(count, b, true); printCmpLenResult(count, expectedCount, b, b); zeroArray(SIZE, c); printDesc("cpu compact without scan, non-power-of-two"); count = StreamCompaction::CPU::compactWithoutScan(NPOT, c, a); expectedNPOT = count; printArray(count, c, true); printCmpLenResult(count, expectedNPOT, b, c); zeroArray(SIZE, c); printDesc("cpu compact with scan"); count = StreamCompaction::CPU::compactWithScan(SIZE, c, a); printArray(count, c, true); printCmpLenResult(count, expectedCount, b, c); zeroArray(SIZE, c); printDesc("work-efficient compact, power-of-two"); count = StreamCompaction::Efficient::compact(SIZE, c, a); //printArray(count, c, true); printCmpLenResult(count, expectedCount, b, c); zeroArray(SIZE, c); printDesc("work-efficient compact, non-power-of-two"); count = StreamCompaction::Efficient::compact(NPOT, c, a); //printArray(count, c, true); printCmpLenResult(count, expectedNPOT, b, c); }
int main(int argc, char* argv[]) { const int SIZE = 1 << 10; const int NPOT = SIZE - 3; int a[SIZE], b[SIZE], c[SIZE]; // Scan tests printf("\n"); printf("****************\n"); printf("** SCAN TESTS **\n"); printf("****************\n"); genArray(SIZE - 1, a, 50); // Leave a 0 at the end to test that edge case a[SIZE - 1] = 0; printArray(SIZE, a, true); zeroArray(SIZE, b); printDesc("cpu scan, power-of-two"); StreamCompaction::CPU::scan(SIZE, b, a); //printArray(SIZE, b, true); zeroArray(SIZE, c); printDesc("cpu scan, non-power-of-two"); StreamCompaction::CPU::scan(NPOT, c, a); //printArray(NPOT, b, true); printCmpResult(NPOT, b, c); zeroArray(SIZE, c); printDesc("naive scan, power-of-two"); StreamCompaction::Naive::scan(SIZE, c, a); //printArray(SIZE, c, true); printCmpResult(SIZE, b, c); zeroArray(SIZE, c); printDesc("naive scan, non-power-of-two"); StreamCompaction::Naive::scan(NPOT, c, a); //printArray(NPOT, c, true); printCmpResult(NPOT, b, c); zeroArray(SIZE, c); printDesc("work-efficient scan, power-of-two"); StreamCompaction::Efficient::scan(SIZE, c, a); //printArray(SIZE, c, true); printCmpResult(SIZE, b, c); zeroArray(SIZE, c); printDesc("work-efficient scan, non-power-of-two"); StreamCompaction::Efficient::scan(NPOT, c, a); //printArray(NPOT, c, true); printCmpResult(NPOT, b, c); zeroArray(SIZE, c); printDesc("thrust scan, power-of-two"); StreamCompaction::Thrust::scan(SIZE, c, a); //printArray(SIZE, c, true); printCmpResult(SIZE, b, c); zeroArray(SIZE, c); printDesc("thrust scan, non-power-of-two"); StreamCompaction::Thrust::scan(NPOT, c, a); //printArray(NPOT, c, true); printCmpResult(NPOT, b, c); //* printf("\n"); printf("*****************************\n"); printf("** STREAM COMPACTION TESTS **\n"); printf("*****************************\n"); // Compaction tests genArray(SIZE - 1, a, 4); // Leave a 0 at the end to test that edge case printArray(SIZE, a, true); a[SIZE - 1] = 0; int count, expectedCount, expectedNPOT; zeroArray(SIZE, b); printDesc("cpu compact without scan, power-of-two"); count = StreamCompaction::CPU::compactWithoutScan(SIZE, b, a); expectedCount = count; printArray(count, b, true); printCmpLenResult(count, expectedCount, b, b); zeroArray(SIZE, c); printDesc("cpu compact without scan, non-power-of-two"); count = StreamCompaction::CPU::compactWithoutScan(NPOT, c, a); expectedNPOT = count; printArray(count, c, true); printCmpLenResult(count, expectedNPOT, b, c); zeroArray(SIZE, c); printDesc("cpu compact with scan"); count = StreamCompaction::CPU::compactWithScan(SIZE, c, a); printArray(count, c, true); printCmpLenResult(count, expectedCount, b, c); zeroArray(SIZE, c); printDesc("work-efficient compact, power-of-two"); printArray(SIZE, a, true);//delete count = StreamCompaction::Efficient::compact(SIZE, c, a); printArray(count, c, true); printCmpLenResult(count, expectedCount, b, c); zeroArray(SIZE, c); printDesc("work-efficient compact, non-power-of-two"); printArray(SIZE, a, true);//delete count = StreamCompaction::Efficient::compact(NPOT, c, a); printArray(count, c, true); printCmpLenResult(count, expectedNPOT, b, c); printf("\n"); printf("*****************************\n"); printf("** Radix Sort **\n"); printf("*****************************\n"); a[0] = 4; a[1] = 7; a[2] = 2; a[3] = 6; a[4] = 3; a[5] = 5; a[6] = 1; a[7] = 0; zeroArray(8, c); printDesc("Radix Sort, power-of-two"); printArray(8, a, true);//delete StreamCompaction::RadixSort::sort(8, c, a); printArray(8, c, true); //printCmpResult(SIZE, b, c);*/ std::cin.get(); }