Exemplo n.º 1
0
void defaultExecution(SysEnv &sysEnv, std::string &metricType, RegionTemplateCollection *rtCollection,
                      std::string tuningPolicy,
                      float *perf, float *totaldiffs, float *metricPerIteration,
                      float *diceNotCoolPerIteration,
                      uint64_t *totalexecutiontimes) {
    int versionNorm = 0, versionSeg = 0;
    int segCount = 0;
    std::vector<int> segComponentIds[rtCollection->getNumRTs()];
    std::vector<int> metricComponentIds[rtCollection->getNumRTs()];
    std::vector<int> diceNotCoolComponentIds[rtCollection->getNumRTs()];
    // Build application dependency graph
    // Instantiate application dependency graph
    for (int i = 0; i < rtCollection->getNumRTs(); i++) {

        int previousSegCompId = 0;
        // CREATE NORMALIZATION STEP
        ParameterSet parSetNormalization;
        std::vector<ArgumentBase *> targetMeanOptions;
        ArgumentFloatArray *targetMeanAux = new ArgumentFloatArray(ArgumentFloat(-0.632356));
        targetMeanAux->addArgValue(ArgumentFloat(-0.0516004));
        targetMeanAux->addArgValue(ArgumentFloat(0.0376543));
        targetMeanOptions.push_back(targetMeanAux);
        parSetNormalization.addArguments(targetMeanOptions);
        parSetNormalization.resetIterator();
        std::vector<ArgumentBase *> argSetInstanceNorm = parSetNormalization.getNextArgumentSetInstance();
        NormalizationComp *norm = new NormalizationComp();
        // normalization parameters
        norm->addArgument(new ArgumentInt(versionNorm));
        norm->addArgument(argSetInstanceNorm[0]);
        norm->addRegionTemplateInstance(rtCollection->getRT(i), rtCollection->getRT(i)->getName());
        sysEnv.executeComponent(norm);


        std::cout << "BEGIN: Default Execution: ";


        std::cout << std::endl;

        // Creating segmentation component
        Segmentation *seg = new Segmentation();

        // version of the data region red. Each parameter instance in norm creates a output w/ different version
        seg->addArgument(new ArgumentInt(versionNorm));
        // version of the data region generated by the segmentation stage
        seg->addArgument(new ArgumentInt(versionSeg));


        int blue = 220;
        int green = 220;
        int red = 220;
        float T1 = 5.0;
        float T2 = 4.0;
        int G1 = 80;
        int minSize = 11;
        int maxSize = 1000;
        int G2 = 45;
        int minSizePl = 30;
        int minSizeSeg = 21;
        int maxSizeSeg = 1000;
        int fillHolesConnectivity = 4;
        int reconConnectivity = 8;
        int watershedConnectivity = 8;

        // add remaining (application specific) parameters from the argSegInstance
        seg->addArgument(new ArgumentInt(
                (int) round(blue)));
        seg->addArgument(new ArgumentInt(
                (int) round(green)));
        seg->addArgument(new ArgumentInt(
                (int) round(red)));
        seg->addArgument(
                new ArgumentFloat((float) (T1)));
        seg->addArgument(
                new ArgumentFloat((float) (T2)));
        seg->addArgument(new ArgumentInt(
                (int) round(G1)));
        seg->addArgument(new ArgumentInt(
                (int) round(G2)));
        seg->addArgument(new ArgumentInt(
                (int) round(minSize)));
        seg->addArgument(new ArgumentInt(
                (int) round(maxSize)));
        seg->addArgument(new ArgumentInt(
                (int) round(minSizePl)));
        seg->addArgument(new ArgumentInt(
                (int) round(minSizeSeg)));
        seg->addArgument(new ArgumentInt(
                (int) round(maxSizeSeg)));
        seg->addArgument(new ArgumentInt(
                (int) round(fillHolesConnectivity)));
        seg->addArgument(new ArgumentInt(
                (int) round(reconConnectivity)));
        seg->addArgument(new ArgumentInt(
                (int) round(watershedConnectivity)));


        // and region template instance that it is suppose to process
        seg->addRegionTemplateInstance(rtCollection->getRT(i), rtCollection->getRT(i)->getName());
        seg->addDependency(norm->getId());

        std::cout << "Creating DiffMask" << std::endl;

        RTPipelineComponentBase *metricComp;
        DiceNotCoolMaskComp *diceNotCoolComp;

        if (metricType.find("jaccard") != std::string::npos) metricComp = new JaccardMaskComp();
        else metricComp = new DiceMaskComp();

        if (metricType.find("dicenc") != std::string::npos) diceNotCoolComp = new DiceNotCoolMaskComp();

        // version of the data region that will be read. It is created during the segmentation.
        // region template name
        metricComp->addArgument(new ArgumentInt(versionSeg));
        metricComp->addRegionTemplateInstance(rtCollection->getRT(i), rtCollection->getRT(i)->getName());
        metricComp->addDependency(seg->getId());


        // add to the list of diff component ids.
        segComponentIds[i].push_back(seg->getId());
        metricComponentIds[i].push_back(metricComp->getId());


        sysEnv.executeComponent(seg);
        sysEnv.executeComponent(metricComp);

        if (metricType.find("dicenc") != std::string::npos) {
            diceNotCoolComp->addArgument(new ArgumentInt(versionSeg));
            diceNotCoolComp->addRegionTemplateInstance(rtCollection->getRT(i),
                                                       rtCollection->getRT(i)->getName());
            diceNotCoolComp->addDependency(metricComp->getId());
            diceNotCoolComponentIds[i].push_back(diceNotCoolComp->getId());
            sysEnv.executeComponent(diceNotCoolComp);
        }


        std::cout << "Manager CompId: " << metricComp->getId() << " fileName: " <<
        rtCollection->getRT(i)->getDataRegion(0)->getInputFileName() << std::endl;
        segCount++;
        versionSeg++;


        versionNorm++;
    }



    // End Creating Dependency Graph
    sysEnv.startupExecution();

    std::cout << std::endl << std::endl;
    //==============================================================================================
    //Fetch results from execution workflow
    //==============================================================================================
    for (int j = 0; j < rtCollection->getNumRTs(); j++) {
        float metric = 0;
        float secondaryMetric = 0;
        float diceNotCoolValue = 0;

        std::ostringstream oss;
        oss << " Default PARAMS";


        for (int i = 0; i < metricComponentIds[j].size(); i++) {
            char *metricResultData = sysEnv.getComponentResultData(metricComponentIds[j][i]);
            std::cout << "RT name: " << rtCollection->getRT(j)->getDataRegion("RAW")->getInputFileName() <<
            " - Diff Id: " << metricComponentIds[j][i];
            if (metricResultData != NULL) {
                //std::cout << "size: " << ((int *) metricResultData)[0] << " \thadoopgis-metric: " <<
                //((float *) metricResultData)[1] <<
                //" \tsecondary: " << ((float *) metricResultData)[2] << endl;

                metric += ((float *) metricResultData)[1];
                secondaryMetric += ((float *) metricResultData)[2];

                if (metricType.find("dicenc") != std::string::npos) {
                    char *diceNotCoolResultData = sysEnv.getComponentResultData(diceNotCoolComponentIds[j][i]);
//                        std::cout << " \tdiceNotCool: " <<
//                        ((float *) diceNotCoolResultData)[1] << std::endl;
                    diceNotCoolValue += ((float *) diceNotCoolResultData)[1];
                }

            } else {
                std::cout << "NULL" << std::endl;
            }
            char *segExecutionTime = sysEnv.getComponentResultData(segComponentIds[j][i]);
            if (segExecutionTime != NULL) {
                totalexecutiontimes[j] = ((int *) segExecutionTime)[1];
//                    cout << "Segmentation execution time:" <<
//                    totalexecutiontimes[j] << endl;
            }
            if (metricType.find("dicenc") != std::string::npos)
                sysEnv.eraseResultData(diceNotCoolComponentIds[j][i]);
            sysEnv.eraseResultData(metricComponentIds[j][i]);
            sysEnv.eraseResultData(segComponentIds[j][i]);
        }
        metricComponentIds[j].clear();
        segComponentIds[j].clear();
        if (metricType.find("dicenc") != std::string::npos) diceNotCoolComponentIds[j].clear();


        float diff;

        if (metricType.find("dicenc") != std::string::npos) diff = (metric + diceNotCoolValue) / 2;
        else diff = metric;

        if (diff <= 0) diff = FLT_EPSILON;


        cout << "\tDiff (average of metric and dnc): " << diff << "\tMetric:" << metric << "\tDNC:" <<
        diceNotCoolValue << " Segmentation Time: " <<
        totalexecutiontimes[j] << endl;

        totaldiffs[j] = diff;
        metricPerIteration[j] = metric;
        diceNotCoolPerIteration[j] = diceNotCoolValue;

    }

    std::cout << std::endl << std::endl;




}
Exemplo n.º 2
0
TuningInterface *multiObjectiveTuning(int argc, char **argv, SysEnv &sysEnv, int max_number_of_tests,
                                      std::string &metricType,
                                      double metricWeight,
                                      double timeWeight,
                                      double &tSlowest, double &tFastest, RegionTemplateCollection *rtCollection,
                                      std::string tuningPolicy,
                                      float *perf, float *totaldiffs, float *metricPerIteration,
                                      float *diceNotCoolPerIteration,
                                      uint64_t *totalexecutiontimes) {

    TuningInterface *tuningClient;
    int numClients;


    //USING AH
    if (tuningPolicy.find("nm") != std::string::npos || tuningPolicy.find("NM") != std::string::npos ||
        tuningPolicy.find("pro") != std::string::npos || tuningPolicy.find("PRO") != std::string::npos ||
        tuningPolicy.find("random") != std::string::npos || tuningPolicy.find("RANDOM") != std::string::npos) {
        numClients = 1;
        tuningClient = new ActiveHarmonyTuning(tuningPolicy, max_number_of_tests, numClients);
    } else {


        //USING GA
        int max_number_of_generations = (int) floor(sqrt(max_number_of_tests));
        int mutationchance = 30;
        int crossoverrate = 50;

        int popsize = (int) ceil(sqrt(max_number_of_tests));
        if (popsize > 1 && ((popsize % 2) == 1)) {
            --popsize;
            max_number_of_generations++;
        } //Pop size must be an even number for GA.

        if (popsize <= 1) popsize++; //pop size must be at least 2.

        if (popsize * max_number_of_generations > max_number_of_tests) {
            max_number_of_generations--;
        }

        //int propagationamount = 2;
        int propagationamount = (int) ceil(popsize * 0.25); //around 25% of popsize

        if ((2 * propagationamount) >= popsize)
            propagationamount--; //Elite size(propagation amount) must be less than half the popsize.

        numClients = popsize; //popsize
        tuningClient = new GeneticAlgorithm(max_number_of_generations, popsize, mutationchance,
                                            crossoverrate,
                                            propagationamount,
                                            1);

    }

    double *totalExecutionTimesNormalized = (double *) malloc(sizeof(double) * max_number_of_tests);

    std::vector<int> segComponentIds[numClients];
    std::vector<int> metricComponentIds[numClients];
    std::vector<int> diceNotCoolComponentIds[numClients];
    std::map<std::string, double> perfDataBase; //Checks if a param has been tested already

    int versionNorm = 0, versionSeg = 0;
    resetPerf(perf, max_number_of_tests);
    bool executedAlready[numClients];
    int repeatCounter = 0;


    if (tuningClient->initialize(argc, argv) != 0) {
        fprintf(stderr, "Failed to initialize tuning session.\n");
        return NULL;
    };


    if (tuningClient->declareParam("blue", 210, 240, 10) != 0 ||
        tuningClient->declareParam("green", 210, 240, 10) != 0 ||
        tuningClient->declareParam("red", 210, 240, 10) != 0 ||
        tuningClient->declareParam("T1", 2.5, 7.5, 0.5) != 0 ||
        tuningClient->declareParam("T2", 2.5, 7.5, 0.5) != 0 ||
        tuningClient->declareParam("G1", 5, 80, 5) != 0 ||
        tuningClient->declareParam("minSize", 2, 40, 2) != 0 ||
        tuningClient->declareParam("maxSize", 900, 1500, 50) != 0 ||
        tuningClient->declareParam("G2", 2, 40, 2) != 0 ||
        tuningClient->declareParam("minSizePl", 5, 80, 5) != 0 ||
        tuningClient->declareParam("minSizeSeg", 2, 40, 2) != 0 ||
        tuningClient->declareParam("maxSizeSeg", 900, 1500, 50) != 0 ||
        tuningClient->declareParam("fillHoles", 4, 8, 4) != 0 ||
        tuningClient->declareParam("recon", 4, 8, 4) != 0 ||
        tuningClient->declareParam("watershed", 4, 8, 4) != 0) {
        fprintf(stderr, "Failed to define tuning session\n");
        return NULL;
    }

    if (tuningClient->configure() != 0) {
        fprintf(stderr, "Failed to initialize tuning session.\n");
        return NULL;
    };

    for (; !tuningClient->hasConverged();) {
        cout << "ITERATION: " << tuningClient->getIteration() << endl;


        //Get new param suggestions from the tuning client
        tuningClient->fetchParams();
        //Apply fitness function for each individual
        for (int i = 0; i < numClients; i++) {

            std::ostringstream oss;
            oss << "PARAMS";
            typedef std::map<std::string, double *>::iterator it_type;
            for (it_type iterator = tuningClient->getParamSet(i)->paramSet.begin();
                 iterator != tuningClient->getParamSet(i)->paramSet.end(); iterator++) {
                //iterator.first key
                //iterator.second value
                oss << " - " << iterator->first << ": " << *(iterator->second);
            }

            // / if not found in performance database
            if (perfDataBase.find(oss.str()) != perfDataBase.end()) {
                perf[tuningClient->getIteration() * numClients +
                     (i)] = perfDataBase.find(oss.str())->second;
                std::cout << "Parameters already tested: " << oss.str() << " perf: " << perf << std::endl;

                executedAlready[i] = true;
            } else {
                executedAlready[i] = false;
            }

        }


        int segCount = 0;
        // Build application dependency graph
        // Instantiate application dependency graph
        for (int i = 0; i < rtCollection->getNumRTs(); i++) {

            int previousSegCompId = 0;
            // CREATE NORMALIZATION STEP
            ParameterSet parSetNormalization;
            std::vector<ArgumentBase *> targetMeanOptions;
            ArgumentFloatArray *targetMeanAux = new ArgumentFloatArray(ArgumentFloat(-0.632356));
            targetMeanAux->addArgValue(ArgumentFloat(-0.0516004));
            targetMeanAux->addArgValue(ArgumentFloat(0.0376543));
            targetMeanOptions.push_back(targetMeanAux);
            parSetNormalization.addArguments(targetMeanOptions);
            parSetNormalization.resetIterator();
            std::vector<ArgumentBase *> argSetInstanceNorm = parSetNormalization.getNextArgumentSetInstance();
            NormalizationComp *norm = new NormalizationComp();
            // normalization parameters
            norm->addArgument(new ArgumentInt(versionNorm));
            norm->addArgument(argSetInstanceNorm[0]);
            norm->addRegionTemplateInstance(rtCollection->getRT(i), rtCollection->getRT(i)->getName());
            sysEnv.executeComponent(norm);

            for (int j = 0; j < numClients; j++) {

                if (executedAlready[j] == false) {

                    std::cout << "BEGIN: LoopIdx: " << tuningClient->getIteration() * numClients + (j);

                    typedef std::map<std::string, double *>::iterator it_type;
                    for (it_type iterator = tuningClient->getParamSet(j)->paramSet.begin();
                         iterator != tuningClient->getParamSet(j)->paramSet.end(); iterator++) {
                        std::cout << " - " << iterator->first << ": " << *(iterator->second);
                    }

                    std::cout << std::endl;

                    // Creating segmentation component
                    Segmentation *seg = new Segmentation();

                    // version of the data region red. Each parameter instance in norm creates a output w/ different version
                    seg->addArgument(new ArgumentInt(versionNorm));
                    // version of the data region generated by the segmentation stage
                    seg->addArgument(new ArgumentInt(versionSeg));

                    // add remaining (application specific) parameters from the argSegInstance
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("blue", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("green", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("red", j))));
                    seg->addArgument(
                            new ArgumentFloat((float) (tuningClient->getParamValue("T1", j))));
                    seg->addArgument(
                            new ArgumentFloat((float) (tuningClient->getParamValue("T2", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("G1", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("G2", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("minSize", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("maxSize", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("minSizePl", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("minSizeSeg", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("maxSizeSeg", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("fillHoles", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("recon", j))));
                    seg->addArgument(new ArgumentInt(
                            (int) round(tuningClient->getParamValue("watershed", j))));


                    // and region template instance that it is suppose to process
                    seg->addRegionTemplateInstance(rtCollection->getRT(i), rtCollection->getRT(i)->getName());
                    seg->addDependency(norm->getId());

                    std::cout << "Creating DiffMask" << std::endl;

                    RTPipelineComponentBase *metricComp;
                    DiceNotCoolMaskComp *diceNotCoolComp;

                    if (metricType.find("jaccard") != std::string::npos) metricComp = new JaccardMaskComp();
                    else metricComp = new DiceMaskComp();

                    if (metricType.find("dicenc") != std::string::npos) diceNotCoolComp = new DiceNotCoolMaskComp();

                    // version of the data region that will be read. It is created during the segmentation.
                    // region template name
                    metricComp->addArgument(new ArgumentInt(versionSeg));
                    metricComp->addRegionTemplateInstance(rtCollection->getRT(i), rtCollection->getRT(i)->getName());
                    metricComp->addDependency(seg->getId());


                    // add to the list of diff component ids.
                    segComponentIds[j].push_back(seg->getId());
                    metricComponentIds[j].push_back(metricComp->getId());


                    sysEnv.executeComponent(seg);
                    sysEnv.executeComponent(metricComp);

                    if (metricType.find("dicenc") != std::string::npos) {
                        diceNotCoolComp->addArgument(new ArgumentInt(versionSeg));
                        diceNotCoolComp->addRegionTemplateInstance(rtCollection->getRT(i),
                                                                   rtCollection->getRT(i)->getName());
                        diceNotCoolComp->addDependency(metricComp->getId());
                        diceNotCoolComponentIds[j].push_back(diceNotCoolComp->getId());
                        sysEnv.executeComponent(diceNotCoolComp);
                    }


                    std::cout << "Manager CompId: " << metricComp->getId() << " fileName: " <<
                    rtCollection->getRT(i)->getDataRegion(0)->getInputFileName() << std::endl;
                    segCount++;
                    versionSeg++;

                }
            }
            versionNorm++;
        }

        // End Creating Dependency Graph
        sysEnv.startupExecution();


        //==============================================================================================
        //Fetch results from execution workflow
        //==============================================================================================
        for (int j = 0; j < numClients; j++) {
            float metric = 0;
            float secondaryMetric = 0;
            float diceNotCoolValue = 0;

            std::ostringstream oss;
            oss << "PARAMS";
            typedef std::map<std::string, double *>::iterator it_type;
            for (it_type iterator = tuningClient->getParamSet(j)->paramSet.begin();
                 iterator != tuningClient->getParamSet(j)->paramSet.end(); iterator++) {
                oss << " - " << iterator->first << ": " << *(iterator->second);
            }

            if (executedAlready[j] == false) {
                for (int i = 0; i < metricComponentIds[j].size(); i++) {
                    char *metricResultData = sysEnv.getComponentResultData(metricComponentIds[j][i]);
                    std::cout << "Diff Id: " << metricComponentIds[j][i] << " \tmetricResultData: ";
                    if (metricResultData != NULL) {
                        std::cout << "size: " << ((int *) metricResultData)[0] << " \thadoopgis-metric: " <<
                        ((float *) metricResultData)[1] <<
                        " \tsecondary: " << ((float *) metricResultData)[2] << endl;

                        metric += ((float *) metricResultData)[1];
                        secondaryMetric += ((float *) metricResultData)[2];

                        if (metricType.find("dicenc") != std::string::npos) {
                            char *diceNotCoolResultData = sysEnv.getComponentResultData(diceNotCoolComponentIds[j][i]);
                            std::cout << " \tdiceNotCool: " <<
                            ((float *) diceNotCoolResultData)[1] << std::endl;
                            diceNotCoolValue += ((float *) diceNotCoolResultData)[1];
                        }

                    } else {
                        std::cout << "NULL" << std::endl;
                    }
                    char *segExecutionTime = sysEnv.getComponentResultData(segComponentIds[j][i]);
                    if (segExecutionTime != NULL) {
                        totalexecutiontimes[tuningClient->getIteration() * numClients +
                                            (j)] = ((int *) segExecutionTime)[1];
                        cout << "Segmentation execution time:" <<
                        totalexecutiontimes[tuningClient->getIteration() * numClients + (j)] << endl;
                    }
                    if (metricType.find("dicenc") != std::string::npos)
                        sysEnv.eraseResultData(diceNotCoolComponentIds[j][i]);
                    sysEnv.eraseResultData(metricComponentIds[j][i]);
                    sysEnv.eraseResultData(segComponentIds[j][i]);
                }
                metricComponentIds[j].clear();
                segComponentIds[j].clear();
                if (metricType.find("dicenc") != std::string::npos) diceNotCoolComponentIds[j].clear();


                //########################################################################################
                //Multi Objective Tuning
                //########################################################################################

                float diff;

                if (metricType.find("dicenc") != std::string::npos) diff = (metric + diceNotCoolValue) / 2;
                else diff = metric;

                if (diff <= 0) diff = FLT_EPSILON;

                double timeNormalized;
                if (timeWeight > 0) {
                    timeNormalized =
                            (tSlowest - (double) totalexecutiontimes[tuningClient->getIteration() * numClients + (j)]) /
                            (tSlowest - tFastest);
//                double timeNormalized = (tSlowest)/
//                        ((double) totalexecutiontimes[tuningClient->getIteration() * numClients + (j)]) ;

                }
                double weightedSumOfMetricAndTime = (double) (metricWeight * diff + timeWeight * timeNormalized);
                if ((weightedSumOfMetricAndTime > 0) && (diff > FLT_EPSILON)) {
                    perf[tuningClient->getIteration() * numClients + (j)] = (double) 1 /
                                                                            weightedSumOfMetricAndTime; //Multi Objective Tuning
                } else {
                    perf[tuningClient->getIteration() * numClients + (j)] = std::numeric_limits<double>::infinity();
                }



                totalExecutionTimesNormalized[tuningClient->getIteration() * numClients + (j)] = timeNormalized;
                if (perf[tuningClient->getIteration() * numClients + (j)] < 0)
                    perf[tuningClient->getIteration() * numClients + (j)] = -perf[
                            tuningClient->getIteration() * numClients + (j)];

                std::cout << "END: LoopIdx: " << tuningClient->getIteration() * numClients + (j);
                typedef std::map<std::string, double *>::iterator it_type;
                for (it_type iterator = tuningClient->getParamSet(j)->paramSet.begin();
                     iterator != tuningClient->getParamSet(j)->paramSet.end(); iterator++) {
                    //iterator.first key
                    //iterator.second value
                    std::cout << " - " << iterator->first << ": " << *(iterator->second);
                }

                cout << endl << endl << "\tDiff: " << diff << " Secondary Metric: " << secondaryMetric <<
                " Time Normalized: " << timeNormalized << " Segmentation Time: " <<
                totalexecutiontimes[tuningClient->getIteration() * numClients + (j)] << " Perf: " <<
                perf[tuningClient->getIteration() * numClients + (j)] << endl;

                totaldiffs[tuningClient->getIteration() * numClients + (j)] = diff;
                metricPerIteration[tuningClient->getIteration() * numClients + (j)] = metric;
                diceNotCoolPerIteration[tuningClient->getIteration() * numClients + (j)] = diceNotCoolValue;


                perfDataBase[oss.str()] = perf[tuningClient->getIteration() * numClients + (j)];
            } else {
                perf[tuningClient->getIteration() * numClients + (j)] = perfDataBase[oss.str()];

                std::cout << "ATTENTION! Param set executed already:" << std::endl;
                std::cout << "END: LoopIdx: " << tuningClient->getIteration() * numClients + (j);
                std::cout << oss.str() << endl;

                std::cout << " perf: " << perf[tuningClient->getIteration() * numClients + (j)] << std::endl;
            }

            // Report the performance we've just measured.
            tuningClient->reportScore(perf[tuningClient->getIteration() * numClients + (j)], j);
        }

        //Checks if at least one test in this iteration succeeded.
        bool shouldIterate = false;
        for (int k = 0; k < numClients; ++k) {
            shouldIterate |= !executedAlready[k];
        }
        //Iterates the tuning algorithm
        if (shouldIterate == true || (repeatCounter >= MAX_ITERATION_REPEAT)) {
            tuningClient->nextIteration();
            repeatCounter = 0;
        } else {
            repeatCounter++;
        }
        //tuningClient->nextIteration(); //Always iterate - to be fair, all 3 algorithms may repeat values.

    }
    return tuningClient;

}