void SyntopReducer::reduce(HadoopPipes::ReduceContext& context) { // NOLINT boost::split(temp_string_components_, context.getInputKey(), boost::is_any_of("_")); float sum = 0; while (context.nextValue()) { sum += HadoopUtils::toFloat(context.getInputValue()); } if (boost::starts_with(temp_string_components_[0], "gamma")) { context.emit(context.getInputKey(), boost::lexical_cast<string>(sum)); } else if (boost::starts_with(temp_string_components_[0], "lhood")) { // sum += GlobalLikelihoodTerm(); sum += GlobalWeightTerm(vars_->nu_.get(), vars_->beta_.get(), params_->alpha_trans(), params_->alpha_top(), params_->finite()); context.emit(context.getInputKey(), boost::lexical_cast<string>(sum)); } else { if (boost::starts_with(temp_string_components_[2], "~")) { // cout << "optimizing" << endl; Optimize(); Emit(&output); StringMap::const_iterator last = (output).end(); for (StringMap::const_iterator itr = (output).begin(); itr != last; itr++) { // cout << itr->first << "\t" << itr->second << endl; context.emit(itr->first, boost::lexical_cast<string>(itr->second)); } output.clear(); last = (output).end(); for (StringMap::const_iterator itr = (output).begin(); itr != last; itr++) { // cout << "output is\t" << itr->first << "\t" << itr->second << endl; // context.emit(itr->first, boost::lexical_cast<string>(itr->second)); } index = boost::lexical_cast<int>(temp_string_components_[1]); vars_ = new VariationalParameters(*params_); display_matrix(vars_->tau_est_top_.get(), "tau_est_top is\n"); display_vector(vars_->tau_est_bottom_.get(), "tau_est_bottom is\n"); tau_coordinate_ = -1; nu_coordinate_ = -1; } else { ProcessKey(context.getInputKey(), sum); // cout << "processing\t" << context.getInputKey() << "\t" << sum << endl; } /* if (index == -1) { index = boost::lexical_cast<double>(temp_string_components_[1]); // reduceContext = context; } else { if (index != boost::lexical_cast<int>(temp_string_components_[1])) { Optimize(); Emit(&output); StringMap::const_iterator last = (output).end(); for (StringMap::const_iterator itr = (output).begin(); itr != last; itr++) { // cout << itr->first << "\t" << itr->second << endl; context.emit(itr->first, boost::lexical_cast<string>(itr->second)); } output.clear(); index = boost::lexical_cast<int>(temp_string_components_[1]); vars_ = new VariationalParameters(*params_); tau_coordinate_ = -1; nu_coordinate_ = -1; } } */ } // } }
/** unittest for oapackage * * Returns UNITTEST_SUCCESS if all tests are ok. * */ int oaunittest (int verbose, int writetests = 0, int randval = 0) { double t0 = get_time_ms (); const char *bstr = "OA unittest"; cprintf (verbose, "%s: start\n", bstr); srand (randval); int allgood = UNITTEST_SUCCESS; Combinations::initialize_number_combinations (20); /* constructors */ { cprintf (verbose, "%s: interaction matrices\n", bstr); array_link al = exampleArray (2); Eigen::MatrixXd m1 = array2xfeigen (al); Eigen::MatrixXd m2 = arraylink2eigen (array2xf (al)); Eigen::MatrixXd dm = m1 - m2; int sum = dm.sum (); myassert (sum == 0, "unittest error: construction of interaction matrices\n"); } cprintf(verbose, "%s: reduceConferenceTransformation\n", bstr); myassert(unittest_reduceConferenceTransformation()==0, "unittest unittest_reduceConferenceTransformation failed"); /* constructors */ { cprintf (verbose, "%s: array manipulation operations\n", bstr); test_array_manipulation (verbose); } /* double conference matrices */ { cprintf (verbose, "%s: double conference matrices\n", bstr); array_link al = exampleArray (36, verbose); myassert (al.is_conference (2), "check on double conference design type"); myassert (testLMC0checkDC (al, verbose >= 2), "testLMC0checkDC"); } /* conference matrices */ { cprintf (verbose, "%s: conference matrices\n", bstr); int N = 4; conference_t ctype (N, N, 0); arraylist_t kk; array_link al = ctype.create_root (); kk.push_back (al); for (int extcol = 2; extcol < N; extcol++) { kk = extend_conference (kk, ctype, 0); } myassert (kk.size () == 1, "unittest error: conference matrices for N=4\n"); } { cprintf (verbose, "%s: generators for conference matrix extensions\n", bstr); test_conference_candidate_generators (verbose); } { cprintf (verbose, "%s: conference matrix Fvalues\n", bstr); array_link al = exampleArray (22, 0); if (verbose >= 2) al.show (); if (0) { std::vector< int > f3 = al.FvaluesConference (3); if (verbose >= 2) { printf ("F3: "); display_vector (f3); printf ("\n"); } } const int N = al.n_rows; jstructconference_t js (N, 4); std::vector< int > f4 = al.FvaluesConference (4); std::vector< int > j4 = js.Jvalues (); if (verbose >= 2) { printf ("j4: "); display_vector (j4); printf ("\n"); printf ("F4: "); display_vector (f4); printf ("\n"); } myassert (j4[0] == 28, "unittest error: conference matricex F values: j4[0]\n"); myassert (f4[0] == 0, "unittest error: conference matricex F values: f4[0] \n"); myassert (f4[1] == 0, "unittest error: conference matricex F values: j4[1]\n"); } { cprintf (verbose, "%s: LMC0 check for arrays in C(4, 3)\n", bstr); array_link al = exampleArray (28, 1); if (verbose >= 2) al.showarray (); lmc_t r = LMC0check (al, verbose); if (verbose >= 2) printf ("LMC0check: result %d\n", r); myassert (r >= LMC_EQUAL, "LMC0 check\n"); al = exampleArray (29, 1); if (verbose >= 2) al.showarray (); r = LMC0check (al, verbose); if (verbose >= 2) printf ("LMC0check: result %d (LMC_LESS %d)\n", r, LMC_LESS); myassert (r == LMC_LESS, "LMC0 check of example array 29\n"); } { cprintf (verbose, "%s: LMC0 check\n", bstr); array_link al = exampleArray (31, 1); if (verbose >= 2) al.showarray (); conference_transformation_t T (al); for (int i = 0; i < 80; i++) { T.randomize (); array_link alx = T.apply (al); lmc_t r = LMC0check (alx, verbose); if (verbose >= 2) { printfd ("%d: transformed array: r %d\n", i, r); alx.showarray (); } if (alx == al) myassert (r >= LMC_EQUAL, "result should be LMC_MORE\n"); else { myassert (r == LMC_LESS, "result should be LMC_LESS\n"); } } } { cprintf (verbose, "%s: random transformation for conference matrices\n", bstr); array_link al = exampleArray (19, 1); conference_transformation_t T (al); // T.randomizerowflips(); T.randomize (); conference_transformation_t Ti = T.inverse (); array_link alx = Ti.apply (T.apply (al)); if (0) { printf ("input array:\n"); al.showarray (); T.show (); printf ("transformed array:\n"); T.apply (al).showarray (); Ti.show (); alx.showarray (); } myassert (alx == al, "transformation of conference matrix\n"); } /* constructors */ { cprintf (verbose, "%s: constructors\n", bstr); array_transformation_t t; conference_transformation_t ct; } /* J-characteristics */ { cprintf (verbose, "%s: J-characteristics\n", bstr); array_link al = exampleArray (8, 1); const int mm[] = {-1, -1, 0, 0, 8, 16, 0, -1}; for (int jj = 2; jj < 7; jj++) { std::vector< int > jx = al.Jcharacteristics (jj); int j5max = vectormax (jx, 0); if (verbose >= 2) { printf ("oaunittest: jj %d: j5max %d\n", jj, j5max); } if (j5max != mm[jj]) { printfd ("j5max %d (should be %d)\n", j5max, mm[jj]); allgood = UNITTEST_FAIL; return allgood; } } } { cprintf (verbose, "%s: array transformations\n", bstr); const int N = 9; const int t = 3; arraydata_t adataX (3, N, t, 4); array_link al (adataX.N, adataX.ncols, -1); al.create_root (adataX); if (checkTransformationInverse (al)) allgood = UNITTEST_FAIL; if (checkTransformationComposition (al, verbose >= 2)) allgood = UNITTEST_FAIL; al = exampleArray (5, 1); if (checkTransformationInverse (al)) allgood = UNITTEST_FAIL; if (checkTransformationComposition (al)) allgood = UNITTEST_FAIL; for (int i = 0; i < 15; i++) { al = exampleArray (18, 0); if (checkConferenceComposition (al)) allgood = UNITTEST_FAIL; if (checkConferenceInverse (al)) allgood = UNITTEST_FAIL; al = exampleArray (19, 0); if (checkConferenceComposition (al)) allgood = UNITTEST_FAIL; if (checkConferenceInverse (al)) allgood = UNITTEST_FAIL; } } { cprintf (verbose, "%s: rank \n", bstr); const int idx[10] = {0, 1, 2, 3, 4, 6, 7, 8, 9}; const int rr[10] = {4, 11, 13, 18, 16, 4, 4, 29, 29}; for (int ii = 0; ii < 9; ii++) { array_link al = exampleArray (idx[ii], 0); myassert (al.is2level (), "unittest error: input array is not 2-level\n"); int r = arrayrankColPivQR (array2xf (al)); int r3 = (array2xf (al)).rank (); myassert (r == r3, "unittest error: rank of array"); if (verbose >= 2) { al.showarray (); printf ("unittest: rank of array %d: %d\n", idx[ii], r); } myassert (rr[ii] == r, "unittest error: rank of example matrix\n"); } } { cprintf (verbose, "%s: Doptimize \n", bstr); const int N = 40; const int t = 0; arraydata_t arrayclass (2, N, t, 6); std::vector< double > alpha (3); alpha[0] = 1; alpha[1] = 1; alpha[2] = 0; int niter = 5000; double t00 = get_time_ms (); DoptimReturn rr = Doptimize (arrayclass, 10, alpha, 0, DOPTIM_AUTOMATIC, niter); array_t ss[7] = {3, 3, 2, 2, 2, 2, 2}; arraydata_t arrayclassmixed (ss, 36, t, 5); rr = Doptimize (arrayclassmixed, 10, alpha, 0, DOPTIM_AUTOMATIC, niter); cprintf (verbose, "%s: Doptimize time %.3f [s] \n", bstr, get_time_ms () - t00); } { cprintf (verbose, "%s: J-characteristics for conference matrix\n", bstr); array_link al = exampleArray (19, 0); std::vector< int > j2 = Jcharacteristics_conference (al, 2); std::vector< int > j3 = Jcharacteristics_conference (al, 3); myassert (j2[0] == 0, "j2 value incorrect"); myassert (j2[1] == 0, "j2 value incorrect"); myassert (std::abs (j3[0]) == 1, "j3 value incorrect"); if (verbose >= 2) { al.showarray (); printf ("j2: "); display_vector (j2); printf ("\n"); printf ("j3: "); display_vector (j3); printf ("\n"); } } { // test PEC sequence cprintf (verbose, "%s: PEC sequence\n", bstr); for (int ii = 0; ii < 5; ii++) { array_link al = exampleArray (ii, 0); std::vector< double > pec = PECsequence (al); printf ("oaunittest: PEC for array %d: ", ii); display_vector (pec); printf (" \n"); } } { cprintf (verbose, "%s: D-efficiency test\n", bstr); // D-efficiency near-zero test { array_link al = exampleArray (14); double D = al.Defficiency (); std::vector< double > dd = al.Defficiencies (); printf ("D %f, D (method 2) %f\n", D, dd[0]); assert (fabs (D - dd[0]) < 1e-4); } { array_link al = exampleArray (15); double D = al.Defficiency (); std::vector< double > dd = al.Defficiencies (); printf ("D %f, D (method 2) %f\n", D, dd[0]); assert (fabs (D - dd[0]) < 1e-4); assert (fabs (D - 0.335063) < 1e-3); } } arraydata_t adata (2, 20, 2, 6); OAextend oaextendx; oaextendx.setAlgorithm ((algorithm_t)MODE_ORIGINAL, &adata); std::vector< arraylist_t > aa (adata.ncols + 1); printf ("OA unittest: create root array\n"); create_root (&adata, aa[adata.strength]); /** Test extend of arrays **/ { cprintf (verbose, "%s: extend arrays\n", bstr); setloglevel (SYSTEM); for (int kk = adata.strength; kk < adata.ncols; kk++) { aa[kk + 1] = extend_arraylist (aa[kk], adata, oaextendx); printf (" extend: column %d->%d: %ld->%ld arrays\n", kk, kk + 1, aa[kk].size (), aa[kk + 1].size ()); } if (aa[adata.ncols].size () != 75) { printf ("extended ?? to %d arrays\n", (int)aa[adata.ncols].size ()); } myassert (aa[adata.ncols].size () == 75, "number of arrays is incorrect"); aa[adata.ncols].size (); setloglevel (QUIET); } { cprintf (verbose, "%s: test LMC check\n", bstr); array_link al = exampleArray (1, 1); lmc_t r = LMCcheckOriginal (al); myassert (r != LMC_LESS, "LMC check of array in normal form"); for (int i = 0; i < 20; i++) { array_link alx = al.randomperm (); if (alx == al) continue; lmc_t r = LMCcheckOriginal (alx); myassert (r == LMC_LESS, "randomized array cannot be in minimal form"); } } { /** Test dof **/ cprintf (verbose, "%s: test delete-one-factor reduction\n", bstr); array_link al = exampleArray (4); cprintf (verbose >= 2, "LMC: \n"); al.reduceLMC (); cprintf (verbose >= 2, "DOP: \n"); al.reduceDOP (); } arraylist_t lst; { /** Test different methods **/ cprintf (verbose, "%s: test 2 different methods\n", bstr); const int s = 2; arraydata_t adata (s, 32, 3, 10); arraydata_t adata2 (s, 32, 3, 10); OAextend oaextendx; oaextendx.setAlgorithm ((algorithm_t)MODE_ORIGINAL, &adata); OAextend oaextendx2; oaextendx2.setAlgorithm ((algorithm_t)MODE_LMC_2LEVEL, &adata2); printf ("OA unittest: test 2-level algorithm on %s\n", adata.showstr ().c_str ()); std::vector< arraylist_t > aa (adata.ncols + 1); create_root (&adata, aa[adata.strength]); std::vector< arraylist_t > aa2 (adata.ncols + 1); create_root (&adata, aa2[adata.strength]); setloglevel (SYSTEM); for (int kk = adata.strength; kk < adata.ncols; kk++) { aa[kk + 1] = extend_arraylist (aa[kk], adata, oaextendx); aa2[kk + 1] = extend_arraylist (aa2[kk], adata2, oaextendx2); printf (" extend: column %d->%d: %ld->%ld arrays, 2-level method %ld->%ld arrays\n", kk, kk + 1, (long) aa[kk].size (), (long)aa[kk + 1].size (), aa2[kk].size (), aa2[kk + 1].size ()); if (aa[kk + 1] != aa2[kk + 1]) { printf ("oaunittest: error: 2-level algorithm unequal to original algorithm\n"); exit (1); } } setloglevel (QUIET); lst = aa[8]; } { cprintf (verbose, "%s: rank calculation using rankStructure\n", bstr); for (int i = 0; i < 27; i++) { array_link al = exampleArray (i, 0); if (al.n_columns < 5) continue; al = exampleArray (i, 1); rankStructure rs; rs.verbose = 0; int r = array2xf (al).rank (); int rc = rs.rankxf (al); if (verbose >= 2) { printf ("rank of example array %d: %d %d\n", i, r, rc); if (verbose >= 3) { al.showproperties (); } } myassert (r == rc, "rank calculations"); } } { cprintf (verbose, "%s: test dtable creation\n", bstr); for (int i = 0; i < 4; i++) { array_link al = exampleArray (5); array_link dtable = createJdtable (al); } } { cprintf (verbose, "%s: test Pareto calculation\n", bstr); double t0x = get_time_ms (); int nn = lst.size (); for (int k = 0; k < 5; k++) { for (int i = 0; i < nn; i++) { lst.push_back (lst[i]); } } Pareto< mvalue_t< long >, long > r = parsePareto (lst, 1); cprintf (verbose, "%s: test Pareto %d/%d: %.3f [s]\n", bstr, r.number (), r.numberindices (), (get_time_ms () - t0x)); } { cprintf (verbose, "%s: check reduction transformation\n", bstr); array_link al = exampleArray (6).reduceLMC (); arraydata_t adata = arraylink2arraydata (al); LMCreduction_t reduction (&adata); reduction.mode = OA_REDUCE; reduction.init_state = COPY; OAextend oaextend; oaextend.setAlgorithm (MODE_ORIGINAL, &adata); array_link alr = al.randomperm (); array_link al2 = reduction.transformation->apply (al); lmc_t tmp = LMCcheck (alr, adata, oaextend, reduction); array_link alx = reduction.transformation->apply (alr); bool c = alx == al; if (!c) { printf ("oaunittest: error: reduction of randomized array failed!\n"); printf ("-- al \n"); al.showarraycompact (); printf ("-- alr \n"); alr.showarraycompact (); printf ("-- alx \n"); alx.showarraycompact (); allgood = UNITTEST_FAIL; } } { cprintf (verbose, "%s: reduce randomized array\n", bstr); array_link al = exampleArray (3); arraydata_t adata = arraylink2arraydata (al); LMCreduction_t reduction (&adata); for (int ii = 0; ii < 50; ii++) { reduction.transformation->randomize (); array_link al2 = reduction.transformation->apply (al); array_link alr = al2.reduceLMC (); if (0) { printf ("\n reduction complete:\n"); al2.showarray (); printf (" --->\n"); alr.showarray (); } bool c = (al == alr); if (!c) { printf ("oaunittest: error: reduction of randomized array failed!\n"); allgood = UNITTEST_FAIL; } } } /* Calculate symmetry group */ { cprintf (verbose, "%s: calculate symmetry group\n", bstr); array_link al = exampleArray (2); symmetry_group sg = al.row_symmetry_group (); assert (sg.permsize () == sg.permsize_large ().toLong ()); // symmetry_group std::vector< int > vv; vv.push_back (0); vv.push_back (0); vv.push_back (1); symmetry_group sg2 (vv); assert (sg2.permsize () == 2); if (verbose >= 2) printf ("sg2: %ld\n", sg2.permsize ()); assert (sg2.ngroups == 2); } /* Test efficiencies */ { cprintf (verbose, "%s: efficiencies\n", bstr); std::vector< double > d; int vb = 1; array_link al; if (1) { al = exampleArray (9, vb); al.showproperties (); d = al.Defficiencies (0, 1); if (verbose >= 2) printf (" efficiencies: D %f Ds %f D1 %f Ds0 %f\n", d[0], d[1], d[2], d[3]); if (fabs (d[0] - al.Defficiency ()) > 1e-10) { printf ("oaunittest: error: Defficiency not good!\n"); allgood = UNITTEST_FAIL; } } al = exampleArray (8, vb); al.showproperties (); d = al.Defficiencies (); if (verbose >= 2) printf (" efficiencies: D %f Ds %f D1 %f\n", d[0], d[1], d[2]); if (fabs (d[0] - al.Defficiency ()) > 1e-10) { printf ("oaunittest: error: Defficiency of examlple array 8 not good!\n"); } al = exampleArray (13, vb); if (verbose >= 3) { al.showarray (); al.showproperties (); } d = al.Defficiencies (0, 1); if (verbose >= 2) printf (" efficiencies: D %f Ds %f D1 %f\n", d[0], d[1], d[2]); if ((fabs (d[0] - 0.939014) > 1e-4) || (fabs (d[3] - 0.896812) > 1e-4) || (fabs (d[2] - 1) > 1e-4)) { printf ("ERROR: D-efficiencies of example array 13 incorrect! \n"); d = al.Defficiencies (2, 1); printf (" efficiencies: D %f Ds %f D1 %f Ds0 %f\n", d[0], d[1], d[2], d[3]); allgood = UNITTEST_FAIL; exit (1); } for (int ii = 11; ii < 11; ii++) { printf ("ii %d: ", ii); al = exampleArray (ii, vb); al.showarray (); al.showproperties (); d = al.Defficiencies (); if (verbose >= 2) printf (" efficiencies: D %f Ds %f D1 %f\n", d[0], d[1], d[2]); } } { cprintf (verbose, "%s: test robustness\n", bstr); array_link A (0, 8, 0); printf ("should return an error\n "); A.Defficiencies (); A = array_link (1, 8, 0); printf ("should return an error\n "); A.at (0, 0) = -2; A.Defficiencies (); } { cprintf (verbose, "%s: test nauty\n", bstr); array_link alr = exampleArray (7, 0); if (unittest_nautynormalform (alr, 1) == 0) { printf ("oaunittest: error: unittest_nautynormalform returns an error!\n"); } } #ifdef HAVE_BOOST if (writetests) { cprintf (verbose, "OA unittest: reading and writing of files\n"); boost::filesystem::path tmpdir = boost::filesystem::temp_directory_path (); boost::filesystem::path temp = boost::filesystem::unique_path ("test-%%%%%%%.oa"); const std::string tempstr = (tmpdir / temp).native (); if (verbose >= 2) printf ("generate text OA file: %s\n", tempstr.c_str ()); int nrows = 16; int ncols = 8; int narrays = 10; arrayfile_t afile (tempstr.c_str (), nrows, ncols, narrays, ATEXT); for (int i = 0; i < narrays; i++) { array_link al (nrows, ncols, array_link::INDEX_DEFAULT); afile.append_array (al); } afile.closefile (); arrayfile_t af (tempstr.c_str (), 0); std::cout << " " << af.showstr () << std::endl; af.closefile (); // check read/write of binary file arraylist_t ll0; ll0.push_back (exampleArray (7)); ll0.push_back (exampleArray (7).randomcolperm ()); writearrayfile (tempstr.c_str (), ll0, ABINARY); arraylist_t ll = readarrayfile (tempstr.c_str ()); myassert (ll0.size () == ll.size (), "read and write of arrays: size of list"); for (size_t i = 0; i < ll0.size (); i++) { myassert (ll0[i] == ll[i], "read and write of arrays: array unequal"); } ll0.resize (0); ll0.push_back (exampleArray (24)); writearrayfile (tempstr.c_str (), ll0, ABINARY_DIFFZERO); ll = readarrayfile (tempstr.c_str ()); myassert (ll0.size () == ll.size (), "read and write of arrays: size of list"); for (size_t i = 0; i < ll0.size (); i++) { myassert (ll0[i] == ll[i], "read and write of arrays: array unequal"); } } #endif { cprintf (verbose, "OA unittest: test nauty\n"); array_link al = exampleArray (5, 2); arraydata_t arrayclass = arraylink2arraydata (al); for (int i = 0; i < 20; i++) { array_link alx = al; alx.randomperm (); array_transformation_t t1 = reduceOAnauty (al); array_link alr1 = t1.apply (al); array_transformation_t t2 = reduceOAnauty (alx); array_link alr2 = t2.apply (alx); if (alr1 != alr2) printf ("oaunittest: error: Nauty reductions unequal!\n"); allgood = UNITTEST_FAIL; } } cprintf (verbose, "OA unittest: complete %.3f [s]!\n", (get_time_ms () - t0)); cprintf (verbose, "OA unittest: also run ptest.py to perform checks!\n"); if (allgood) { printf ("OA unittest: all tests ok\n"); return UNITTEST_SUCCESS; } else { printf ("OA unittest: ERROR!\n"); return UNITTEST_FAIL; } }