TEST_F(AnalysisDriverFixture,SimpleProject_EditProblemWithTwoWorkflows) { // Written to reproduce Bug 1189 { // Create PAT project with an IDF measure SimpleProject project = getCleanPATProject("SimpleProject_EditProblemWithTwoWorkflows"); Problem problem = project.analysis().problem(); OptionalInt index = problem.getWorkflowStepIndexByJobType(JobType::EnergyPlusPreProcess); ASSERT_TRUE(index); MeasureGroup dv("Idf Measure",MeasureVector(1u,NullMeasure())); problem.insert(*index,dv); BOOST_FOREACH(const BCLMeasure& measure,BCLMeasure::patApplicationMeasures()) { if (measure.inputFileType() == FileReferenceType::IDF) { RubyMeasure measure(measure); dv.push(measure); break; } } project.save(); ProjectDatabase database = project.projectDatabase(); RunManager runManager = project.runManager(); WorkflowRecordVector workflowRecords = WorkflowRecord::getWorkflowRecords(database); EXPECT_EQ(2u,workflowRecords.size()); BOOST_FOREACH(const WorkflowRecord& wr,workflowRecords) { EXPECT_NO_THROW(runManager.loadWorkflow(wr.runManagerWorkflowKey())); } }
TEST_F(AnalysisDriverFixture,SimpleProject_NonPATToPATProject) { { // create project with one discrete variable SimpleProject project = getCleanSimpleProject("SimpleProject_NonPATToPATProject"); openstudio::path measuresDir = resourcesPath() / toPath("/utilities/BCL/Measures"); openstudio::path dir = measuresDir / toPath("SetWindowToWallRatioByFacade"); BCLMeasure measure = BCLMeasure::load(dir).get(); RubyMeasure rmeasure(measure); project.analysis().problem().push(MeasureGroup("My Variable",MeasureVector(1u,rmeasure))); DataPoint baseline = project.baselineDataPoint(); EXPECT_EQ(1u,baseline.variableValues().size()); EXPECT_TRUE(project.analysis().isDirty()); EXPECT_EQ(1u,project.analysis().dataPoints().size()); project.save(); } { SimpleProject project = getPATProject("SimpleProject_NonPATToPATProject"); DataPoint baseline = project.baselineDataPoint(); EXPECT_EQ(2u,baseline.variableValues().size()); EXPECT_FALSE(project.analysis().isDirty()); // openPATProject calls save in this case EXPECT_EQ(1u,project.analysis().dataPoints().size()); project.save(); } { SimpleProject project = getPATProject("SimpleProject_NonPATToPATProject"); DataPoint baseline = project.baselineDataPoint(); EXPECT_EQ(2u,baseline.variableValues().size()); EXPECT_FALSE(project.analysis().isDirty()); // nothing to change in this case EXPECT_EQ(1u,project.analysis().dataPoints().size()); } }
TEST_F(AnalysisDriverFixture,SimpleProject_InsertMeasure) { // create a new project SimpleProject project = getCleanSimpleProject("SimpleProject_InsertMeasure"); // open a measure openstudio::path dir = resourcesPath() / toPath("/utilities/BCL/Measures/SetWindowToWallRatioByFacade"); BCLMeasure measure = BCLMeasure::load(dir).get(); // insert it into the project BCLMeasure projectMeasure = project.insertMeasure(measure); // verify that there is a project-specific copy EXPECT_NE(toString(measure.directory()),toString(projectMeasure.directory())); openstudio::path scriptsDir = project.projectDir() / toPath("scripts"); EXPECT_EQ(toString(completeAndNormalize(scriptsDir)), toString(completeAndNormalize(projectMeasure.directory().parent_path()))); EXPECT_EQ(measure.uuid(),projectMeasure.uuid()); EXPECT_EQ(measure.versionUUID(),projectMeasure.versionUUID()); // DLM: should this be not equal? // verify that it can be retrieved from the project, but its arguments cannot ASSERT_TRUE(project.getMeasureByUUID(measure.uuid())); BCLMeasure temp = project.getMeasureByUUID(measure.uuid()).get(); EXPECT_EQ(toString(projectMeasure.directory()),toString(temp.directory())); EXPECT_EQ(projectMeasure.uuid(),temp.uuid()); EXPECT_EQ(projectMeasure.versionUUID(),temp.versionUUID()); EXPECT_EQ(1u,project.measures().size()); // use embedded Ruby to extract the arguments // (see AnalysisDriverFixture.cpp for interpreter set-up) OSArgumentVector args = argumentGetter->getArguments(projectMeasure, project.seedModel(), project.seedIdf()); EXPECT_FALSE(args.empty()); // register and subsequently retrieve those arguments project.registerArguments(measure,args); // doesn't matter which copy of the measure we use EXPECT_TRUE(project.hasStoredArguments(temp)); OSArgumentVector argsCopy = project.getStoredArguments(projectMeasure); ASSERT_EQ(args.size(),argsCopy.size()); for (int i = 0, n = args.size(); i < n; ++i) { EXPECT_EQ(args[i].name(),argsCopy[i].name()); EXPECT_EQ(args[i].type(),argsCopy[i].type()); } // use this measure to create a new variable/ruby measure // now it is important to use the copy in the project MeasureGroup dv("New Measure Group",MeasureVector()); Problem problem = project.analysis().problem(); problem.push(dv); // here, expect this test to pass. // in pat, if it fails, let user know and call problem.erase(dv). EXPECT_TRUE(problem.fileTypesAreCompatible(dv, projectMeasure.inputFileType(), projectMeasure.outputFileType())); EXPECT_TRUE(dv.push(RubyMeasure(projectMeasure))); }
TEST_F(AnalysisDriverFixture,PATProject_CreateOpenAndSaveAs) { openstudio::path projectDir; { // Create a PAT project SimpleProject patProject = getCleanPATProject("PATProject_CreateOpenAndSaveAs"); // Populate it patProject.analysis().setName("PAT Project: Create, Open, and Save As"); Model model = model::exampleModel(); openstudio::path p = toPath("./example.osm"); model.save(p,true); FileReference seedModel(p); patProject.setSeed(seedModel); // (feature not in PAT, but not incompatible) DesignOfExperimentsOptions algOptions(DesignOfExperimentsType::FullFactorial); DesignOfExperiments algorithm(algOptions); patProject.analysis().setAlgorithm(algorithm); // Save it patProject.save(); projectDir = patProject.projectDir(); } // Open Project OptionalSimpleProject project = openPATProject(projectDir); ASSERT_TRUE(project); EXPECT_TRUE(project->isPATProject()); // Save Project As openstudio::path newProjectDir = toPath("AnalysisDriverFixtureData") / toPath("PATProjectCreateOpenAndSaveAsCopy"); boost::filesystem::remove_all(newProjectDir); ASSERT_TRUE(project->saveAs(newProjectDir)); OptionalSimpleProject projectCopy = openPATProject(newProjectDir); ASSERT_TRUE(projectCopy); EXPECT_TRUE(project->isPATProject()); EXPECT_TRUE(project->analysis().algorithm()); }
TEST_F(AnalysisDriverFixture, DesignOfExperiments_MeshAnalysis) { openstudio::path rubyLibDirPath = openstudio::toPath(rubyLibDir()); // GET SIMPLE PROJECT SimpleProject project = getCleanSimpleProject("DesignOfExperiments_MeshAnalysis"); Analysis analysis = project.analysis(); // SET PROBLEM Problem problem = retrieveProblem("MixedOsmIdf",false,false); analysis.setProblem(problem); // SET SEED Model model = model::exampleModel(); openstudio::path p = toPath("./example.osm"); model.save(p,true); FileReference seedModel(p); analysis.setSeed(seedModel); // SET ALGORITHM DesignOfExperimentsOptions algOptions(DesignOfExperimentsType::FullFactorial); DesignOfExperiments algorithm(algOptions); analysis.setAlgorithm(algorithm); // RUN ANALYSIS AnalysisDriver driver = project.analysisDriver(); AnalysisRunOptions runOptions = standardRunOptions(project.projectDir()); driver.run(analysis,runOptions); EXPECT_TRUE(driver.waitForFinished()); // CHECK RESULTS AnalysisRecord analysisRecord = project.analysisRecord(); EXPECT_EQ(4,analysisRecord.problemRecord().combinatorialSize(true).get()); EXPECT_EQ(4u, analysisRecord.dataPointRecords().size()); BOOST_FOREACH(const DataPointRecord& dataPointRecord, analysisRecord.dataPointRecords()) { EXPECT_TRUE(dataPointRecord.isComplete()); EXPECT_FALSE(dataPointRecord.failed()); } // get data points by perturbations and vice versa std::vector<DataPointRecord> testDataPoints; std::vector<QVariant> testVariableValues; // all data points are successful testDataPoints = analysisRecord.successfulDataPointRecords(); EXPECT_EQ(4u,testDataPoints.size()); // empty variableValues returns all data points testDataPoints = analysisRecord.getDataPointRecords(testVariableValues); EXPECT_EQ(4u, testDataPoints.size()); // find the baseline testVariableValues.clear(); testVariableValues.push_back(0); testVariableValues.push_back(QVariant(QVariant::Int)); // only one perturbation, null works too testVariableValues.push_back(0); ASSERT_TRUE(testVariableValues[1].isNull()); testDataPoints = analysisRecord.getDataPointRecords(testVariableValues); ASSERT_EQ(1u, testDataPoints.size()); // find model with improved wall and roof testVariableValues.clear(); testVariableValues.push_back(1); testVariableValues.push_back(0); testVariableValues.push_back(1); testDataPoints = analysisRecord.getDataPointRecords(testVariableValues); ASSERT_EQ(1u, testDataPoints.size()); DataPoint testDataPoint = testDataPoints[0].dataPoint(); std::vector<OptionalDiscretePerturbation> perturbations = analysis.problem().getDiscretePerturbations(testVariableValues); ASSERT_EQ(3u,perturbations.size()); ASSERT_TRUE(perturbations[0] && perturbations[1] && perturbations[2]); EXPECT_TRUE(perturbations[0]->uuid() == problem.variables()[0].cast<DiscreteVariable>().perturbations(false)[1].uuid()); EXPECT_TRUE(perturbations[1]->uuid() == problem.variables()[1].cast<DiscreteVariable>().perturbations(false)[0].uuid()); EXPECT_TRUE(perturbations[2]->uuid() == problem.variables()[2].cast<DiscreteVariable>().perturbations(false)[1].uuid()); EXPECT_TRUE(perturbations[0]->optionalCast<RubyPerturbation>()); EXPECT_TRUE(perturbations[1]->optionalCast<RubyPerturbation>()); EXPECT_TRUE(perturbations[2]->optionalCast<RubyPerturbation>()); // find models with improved wall testVariableValues.clear(); testVariableValues.push_back(1); testDataPoints = analysisRecord.getDataPointRecords(testVariableValues); ASSERT_EQ(2u, testDataPoints.size()); // infeasible testVariableValues.clear(); testVariableValues.push_back(0); testVariableValues.push_back(0); testVariableValues.push_back(0); testVariableValues.push_back(0); testDataPoints = analysisRecord.getDataPointRecords(testVariableValues); ASSERT_EQ(0u, testDataPoints.size()); }
TEST_F(AnalysisDriverFixture, DDACE_LatinHypercube_MixedOsmIdf_MoveProjectDatabase) { openstudio::path oldDir, newDir; { // GET SIMPLE PROJECT SimpleProject project = getCleanSimpleProject("DDACE_LatinHypercube_MixedOsmIdf"); Analysis analysis = project.analysis(); analysis.setName("DDACE Latin Hypercube Sampling - MixedOsmIdf"); // SET PROBLEM Problem problem = retrieveProblem("MixedOsmIdf",false,false); analysis.setProblem(problem); // SET SEED Model model = model::exampleModel(); openstudio::path p = toPath("./example.osm"); model.save(p,true); FileReference seedModel(p); analysis.setSeed(seedModel); // SET ALGORITHM DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::lhs); algOptions.setSamples(12); // test reprinting results.out for copies of same point DDACEAlgorithm algorithm(algOptions); analysis.setAlgorithm(algorithm); // RUN ANALYSIS AnalysisDriver driver = project.analysisDriver(); AnalysisRunOptions runOptions = standardRunOptions(project.projectDir()); CurrentAnalysis currentAnalysis = driver.run(analysis,runOptions); EXPECT_TRUE(driver.waitForFinished()); boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors(); ASSERT_TRUE(jobErrors); EXPECT_TRUE(jobErrors->errors().empty()); EXPECT_TRUE(driver.currentAnalyses().empty()); Table summary = analysis.summaryTable(); EXPECT_EQ(5u,summary.nRows()); // 4 points (all combinations) summary.save(project.projectDir() / toPath("summary.csv")); EXPECT_EQ(4u,analysis.dataPoints().size()); BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) { EXPECT_TRUE(dataPoint.isComplete()); EXPECT_FALSE(dataPoint.failed()); EXPECT_TRUE(dataPoint.workspace()); // should be able to load data from disk } oldDir = project.projectDir(); newDir = project.projectDir().parent_path() / toPath("DDACELatinHypercubeMixedOsmIdfCopy"); // Make copy of project boost::filesystem::remove_all(newDir); ASSERT_TRUE(project.saveAs(newDir)); } // Blow away old project. // TODO: Reinstate. This was failing on Windows and isn't absolutely necessary. // try { // boost::filesystem::remove_all(oldDir); // } // catch (std::exception& e) { // EXPECT_TRUE(false) << "Boost filesystem was unable to delete the old folder, because " << e.what(); // } // Open new project SimpleProject project = getSimpleProject("DDACE_LatinHypercube_MixedOsmIdf_Copy"); EXPECT_TRUE(project.projectDir() == newDir); EXPECT_EQ(toString(newDir),toString(project.projectDir())); // After move, should be able to retrieve results. EXPECT_FALSE(project.analysisIsLoaded()); Analysis analysis = project.analysis(); EXPECT_TRUE(project.analysisIsLoaded()); EXPECT_EQ(4u,analysis.dataPoints().size()); BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) { EXPECT_TRUE(dataPoint.isComplete()); EXPECT_FALSE(dataPoint.failed()); LOG(Debug,"Attempting to load workspace for data point at '" << dataPoint.directory() << "'."); if (dataPoint.idfInputData()) { LOG(Debug,"Says there should be input data at " << toString(dataPoint.idfInputData()->path())); } EXPECT_TRUE(dataPoint.workspace()); // should be able to load data from disk if (!dataPoint.workspace()) { LOG(Debug,"Unsuccessful.") } } // Should be able to blow away results and run again project.removeAllDataPoints(); EXPECT_EQ(0u,analysis.dataPoints().size()); EXPECT_FALSE(analysis.algorithm()->isComplete()); EXPECT_FALSE(analysis.algorithm()->failed()); EXPECT_EQ(-1,analysis.algorithm()->iter()); EXPECT_FALSE(analysis.algorithm()->cast<DakotaAlgorithm>().restartFileReference()); EXPECT_FALSE(analysis.algorithm()->cast<DakotaAlgorithm>().outFileReference()); AnalysisRunOptions runOptions = standardRunOptions(project.projectDir()); AnalysisDriver driver = project.analysisDriver(); CurrentAnalysis currentAnalysis = driver.run(analysis,runOptions); EXPECT_TRUE(driver.waitForFinished()); boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors(); ASSERT_TRUE(jobErrors); EXPECT_TRUE(jobErrors->errors().empty()); EXPECT_TRUE(driver.currentAnalyses().empty()); Table summary = analysis.summaryTable(); EXPECT_EQ(5u,summary.nRows()); // 4 points (all combinations) summary.save(project.projectDir() / toPath("summary.csv")); BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) { EXPECT_TRUE(dataPoint.isComplete()); EXPECT_FALSE(dataPoint.failed()); EXPECT_TRUE(dataPoint.workspace()); // should be able to load data from disk } }
TEST_F(AnalysisDriverFixture, DDACE_LatinHypercube_Continuous) { { // GET SIMPLE PROJECT SimpleProject project = getCleanSimpleProject("DDACE_LatinHypercube_Continuous"); Analysis analysis = project.analysis(); // SET PROBLEM Problem problem = retrieveProblem("Continuous",true,false); analysis.setProblem(problem); // DEFINE SEED Model model = model::exampleModel(); openstudio::path p = toPath("./example.osm"); model.save(p,true); FileReference seedModel(p); analysis.setSeed(seedModel); // CREATE ANALYSIS DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::lhs); DDACEAlgorithm algorithm(algOptions); analysis.setAlgorithm(algorithm); // RUN ANALYSIS AnalysisDriver driver = project.analysisDriver(); AnalysisRunOptions runOptions = standardRunOptions(project.projectDir()); CurrentAnalysis currentAnalysis = driver.run(analysis,runOptions); EXPECT_TRUE(driver.waitForFinished()); boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors(); ASSERT_TRUE(jobErrors); EXPECT_FALSE(jobErrors->errors().empty()); // require specification of number of samples EXPECT_TRUE(driver.currentAnalyses().empty()); Table summary = currentAnalysis.analysis().summaryTable(); EXPECT_EQ(1u,summary.nRows()); // no points project.clearAllResults(); algOptions.setSamples(4); EXPECT_EQ(4,analysis.algorithm()->cast<DDACEAlgorithm>().ddaceAlgorithmOptions().samples()); currentAnalysis = driver.run(analysis,runOptions); EXPECT_TRUE(driver.waitForFinished()); jobErrors = currentAnalysis.dakotaJobErrors(); ASSERT_TRUE(jobErrors); EXPECT_TRUE(jobErrors->errors().empty()); EXPECT_TRUE(driver.currentAnalyses().empty()); summary = currentAnalysis.analysis().summaryTable(); EXPECT_EQ(5u,summary.nRows()); summary.save(project.projectDir() / toPath("summary.csv")); BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) { EXPECT_TRUE(dataPoint.isComplete()); EXPECT_FALSE(dataPoint.failed()); // EXPECT_FALSE(dataPoint.responseValues().empty()); } ASSERT_TRUE(analysis.algorithm()); EXPECT_TRUE(analysis.algorithm()->isComplete()); EXPECT_FALSE(analysis.algorithm()->failed()); { AnalysisRecord analysisRecord = project.analysisRecord(); Analysis analysisCopy = analysisRecord.analysis(); ASSERT_TRUE(analysisCopy.algorithm()); EXPECT_TRUE(analysisCopy.algorithm()->isComplete()); EXPECT_FALSE(analysisCopy.algorithm()->failed()); } } LOG(Info,"Restart from existing project."); // Get existing project SimpleProject project = getSimpleProject("DDACE_LatinHypercube_Continuous"); EXPECT_FALSE(project.analysisIsLoaded()); // make sure starting fresh Analysis analysis = project.analysis(); EXPECT_FALSE(analysis.isDirty()); // Add custom data point std::vector<QVariant> values; values.push_back(0.0); values.push_back(0.8); values.push_back(int(0)); OptionalDataPoint dataPoint = analysis.problem().createDataPoint(values); ASSERT_TRUE(dataPoint); analysis.addDataPoint(*dataPoint); EXPECT_EQ(1u,analysis.dataPointsToQueue().size()); ASSERT_TRUE(analysis.algorithm()); EXPECT_TRUE(analysis.algorithm()->isComplete()); EXPECT_FALSE(analysis.algorithm()->failed()); EXPECT_TRUE(analysis.isDirty()); EXPECT_FALSE(analysis.resultsAreInvalid()); EXPECT_FALSE(analysis.dataPointsAreInvalid()); // get last modified time of a file in a completed data point to make sure nothing is re-run DataPointVector completePoints = analysis.completeDataPoints(); ASSERT_FALSE(completePoints.empty()); OptionalFileReference inputFileRef = completePoints[0].osmInputData(); ASSERT_TRUE(inputFileRef); QFileInfo inputFileInfo(toQString(inputFileRef->path())); QDateTime inputFileModifiedTestTime = inputFileInfo.lastModified(); EXPECT_EQ(1u,analysis.dataPointsToQueue().size()); AnalysisDriver driver = project.analysisDriver(); CurrentAnalysis currentAnalysis = driver.run( analysis, standardRunOptions(project.projectDir())); EXPECT_TRUE(driver.waitForFinished()); boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors(); EXPECT_FALSE(jobErrors); // should not try to re-run DakotaAlgorithm EXPECT_TRUE(driver.currentAnalyses().empty()); EXPECT_TRUE(analysis.dataPointsToQueue().empty()); Table summary = currentAnalysis.analysis().summaryTable(); EXPECT_EQ(6u,summary.nRows()); summary.save(project.projectDir() / toPath("summary_post_restart.csv")); // RunManager should not re-run any data points EXPECT_EQ(inputFileModifiedTestTime,inputFileInfo.lastModified()); }
TEST_F(AnalysisDriverFixture,SimpleProject_UpdateMeasure) { // create a new project SimpleProject project = getCleanSimpleProject("SimpleProject_UpdateMeasure"); Problem problem = project.analysis().problem(); // insert a measure into the project, extract and register its arguments openstudio::path measuresDir = resourcesPath() / toPath("/utilities/BCL/Measures"); openstudio::path dir = measuresDir / toPath("SetWindowToWallRatioByFacade"); BCLMeasure measure = BCLMeasure::load(dir).get(); BCLMeasure projectMeasure = project.insertMeasure(measure); OSArgumentVector args = argumentGetter->getArguments(projectMeasure, project.seedModel(), project.seedIdf()); project.registerArguments(projectMeasure,args); EXPECT_EQ(1u,project.measures().size()); // use the measure to create a new variable/ruby measure MeasureGroup dv("New Measure Group",MeasureVector()); EXPECT_TRUE(problem.push(dv)); RubyMeasure rp(projectMeasure); rp.setArguments(args); EXPECT_TRUE(dv.push(rp)); EXPECT_EQ(args.size(),rp.arguments().size()); EXPECT_TRUE(rp.hasIncompleteArguments()); BOOST_FOREACH(const OSArgument& arg,args) { if (arg.name() == "wwr") { OSArgument temp = arg.clone(); temp.setValue(0.6); rp.setArgument(temp); } if (arg.name() == "sillHeight") { OSArgument temp = arg.clone(); temp.setValue(1.0); rp.setArgument(temp); } if (arg.name() == "facade") { OSArgument temp = arg.clone(); temp.setValue("South"); rp.setArgument(temp); } } EXPECT_FALSE(rp.hasIncompleteArguments()); openstudio::path tempDir = measuresDir / toPath(toString(createUUID())); { // create fake new version of the measure BCLMeasure newVersion = measure.clone(tempDir).get(); newVersion.incrementVersionId(); newVersion.save(); OSArgumentVector newArgs = args; newArgs.push_back(OSArgument::makeDoubleArgument("frame_width")); // update the measure project.updateMeasure(newVersion,newArgs); // verify the final state of SimpleProject and RubyMeasure EXPECT_EQ(1u,project.measures().size()); BCLMeasure retrievedMeasure = project.getMeasureByUUID(measure.uuid()).get(); EXPECT_NE(measure.versionUUID(),retrievedMeasure.versionUUID()); EXPECT_EQ(newVersion.versionUUID(),retrievedMeasure.versionUUID()); ASSERT_TRUE(project.hasStoredArguments(retrievedMeasure)); OSArgumentVector retrievedArgs = project.getStoredArguments(retrievedMeasure); EXPECT_EQ(args.size() + 1u,retrievedArgs.size()); EXPECT_EQ(retrievedArgs.size(),rp.arguments().size()); EXPECT_TRUE(rp.hasIncompleteArguments()); } boost::filesystem::remove_all(tempDir); }
TEST_F(AnalysisDriverFixture,DataPersistence_DataPointErrors) { { // Create and populate project SimpleProject project = getCleanSimpleProject("DataPersistence_DataPointErrors"); Analysis analysis = project.analysis(); Problem problem = retrieveProblem(AnalysisDriverFixtureProblem::BuggyBCLMeasure, true, false); EXPECT_EQ(5u,problem.workflow().size()); analysis.setProblem(problem); model::Model model =fastExampleModel(); openstudio::path p = toPath("./example.osm"); model.save(p,true); FileReference seedModel(p); project.setSeed(seedModel); DataPoint dataPoint = problem.createDataPoint(std::vector<QVariant>(problem.numVariables(),0)).get(); analysis.addDataPoint(dataPoint); // Run analysis AnalysisRunOptions runOptions = standardRunOptions(project.projectDir()); project.analysisDriver().run(analysis,runOptions); project.analysisDriver().waitForFinished(); // Check DataPoint job and error information ASSERT_EQ(1u,analysis.dataPoints().size()); dataPoint = analysis.dataPoints()[0]; EXPECT_TRUE(dataPoint.isComplete()); EXPECT_TRUE(dataPoint.failed()); EXPECT_TRUE(dataPoint.topLevelJob()); WorkflowStepJobVector jobResults = problem.getJobsByWorkflowStep(dataPoint); EXPECT_EQ(problem.workflow().size(),jobResults.size()); ASSERT_EQ(5u,jobResults.size()); WorkflowStepJob jobResult = jobResults[0]; ASSERT_TRUE(jobResult.job); EXPECT_TRUE(jobResult.measure); Job job = jobResult.job.get(); ASSERT_TRUE(jobResult.mergedJobIndex); EXPECT_EQ(0u,jobResult.mergedJobIndex.get()); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); JobErrors treeErrors = job.treeErrors(); // get all tree errors now, test later JobErrors errors = jobResult.errors().get(); EXPECT_EQ(OSResultValue(OSResultValue::NA),errors.result); EXPECT_TRUE(errors.succeeded()); EXPECT_TRUE(errors.errors().empty()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_FALSE(errors.infos().empty()); jobResult = jobResults[1]; ASSERT_TRUE(jobResult.job); EXPECT_FALSE(jobResult.measure); ASSERT_TRUE(jobResult.step.isWorkItem()); EXPECT_EQ(JobType(JobType::UserScript),jobResult.step.workItemType()); job = jobResult.job.get(); ASSERT_TRUE(jobResult.mergedJobIndex); EXPECT_EQ(1u,jobResult.mergedJobIndex.get()); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); errors = jobResult.errors().get(); EXPECT_EQ(OSResultValue(OSResultValue::Success),errors.result); EXPECT_TRUE(errors.succeeded()); EXPECT_TRUE(errors.errors().empty()); EXPECT_FALSE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); jobResult = jobResults[2]; ASSERT_TRUE(jobResult.job); EXPECT_TRUE(jobResult.measure); job = jobResult.job.get(); ASSERT_TRUE(jobResult.mergedJobIndex); EXPECT_EQ(2u,jobResult.mergedJobIndex.get()); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); errors = jobResult.errors().get(); EXPECT_EQ(OSResultValue(OSResultValue::Fail),errors.result); EXPECT_FALSE(errors.succeeded()); EXPECT_FALSE(errors.errors().empty()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); jobResult = jobResults[3]; ASSERT_TRUE(jobResult.job); EXPECT_FALSE(jobResult.measure); ASSERT_TRUE(jobResult.step.isWorkItem()); EXPECT_EQ(JobType(JobType::UserScript),jobResult.step.workItemType()); job = jobResult.job.get(); ASSERT_TRUE(jobResult.mergedJobIndex); EXPECT_EQ(3u,jobResult.mergedJobIndex.get()); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); // now all four scripts are in same job EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); // now all four scripts are in same job errors = jobResult.errors().get(); // this script not actually run, so result in default state EXPECT_EQ(OSResultValue(OSResultValue::Fail),errors.result); EXPECT_FALSE(errors.succeeded()); EXPECT_EQ(1u, errors.errors().size()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); jobResult = jobResults[4]; ASSERT_TRUE(jobResult.job); EXPECT_FALSE(jobResult.measure); ASSERT_TRUE(jobResult.step.isWorkItem()); EXPECT_EQ(JobType(JobType::ModelToIdf),jobResult.step.workItemType()); job = jobResult.job.get(); EXPECT_FALSE(jobResult.mergedJobIndex); EXPECT_TRUE(job.outOfDate()); // never run EXPECT_FALSE(job.canceled()); EXPECT_FALSE(job.lastRun()); errors = jobResult.errors().get(); EXPECT_EQ(OSResultValue(OSResultValue::Fail),errors.result); EXPECT_FALSE(errors.succeeded()); EXPECT_TRUE(errors.errors().empty()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); EXPECT_EQ(OSResultValue(OSResultValue::Fail),treeErrors.result); EXPECT_FALSE(treeErrors.succeeded()); EXPECT_FALSE(treeErrors.errors().empty()); EXPECT_FALSE(treeErrors.warnings().empty()); EXPECT_FALSE(treeErrors.infos().empty()); } { // Re-open project SimpleProject project = getSimpleProject("DataPersistence_DataPointErrors"); Analysis analysis = project.analysis(); Problem problem = analysis.problem(); // Verify job and error information still there // Check DataPoint job and error information ASSERT_EQ(1u,analysis.dataPoints().size()); DataPoint dataPoint = analysis.dataPoints()[0]; EXPECT_TRUE(dataPoint.isComplete()); EXPECT_TRUE(dataPoint.failed()); EXPECT_TRUE(dataPoint.topLevelJob()); WorkflowStepJobVector jobResults = problem.getJobsByWorkflowStep(dataPoint); EXPECT_EQ(problem.workflow().size(),jobResults.size()); ASSERT_EQ(5u,jobResults.size()); WorkflowStepJob jobResult = jobResults[0]; ASSERT_TRUE(jobResult.job); EXPECT_TRUE(jobResult.measure); Job job = jobResult.job.get(); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); JobErrors treeErrors = job.treeErrors(); // get all tree errors now, test later JobErrors errors = jobResult.errors().get(); EXPECT_EQ(OSResultValue(OSResultValue::NA),errors.result); EXPECT_TRUE(errors.succeeded()); EXPECT_TRUE(errors.errors().empty()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_FALSE(errors.infos().empty()); jobResult = jobResults[1]; ASSERT_TRUE(jobResult.job); EXPECT_FALSE(jobResult.measure); ASSERT_TRUE(jobResult.step.isWorkItem()); EXPECT_EQ(JobType(JobType::UserScript),jobResult.step.workItemType()); job = jobResult.job.get(); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); errors = jobResult.errors().get(); EXPECT_EQ(OSResultValue(OSResultValue::Success),errors.result); EXPECT_TRUE(errors.succeeded()); EXPECT_TRUE(errors.errors().empty()); EXPECT_FALSE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); jobResult = jobResults[2]; ASSERT_TRUE(jobResult.job); EXPECT_TRUE(jobResult.measure); job = jobResult.job.get(); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); errors = jobResult.errors().get(); EXPECT_EQ(OSResultValue(OSResultValue::Fail),errors.result); EXPECT_FALSE(errors.succeeded()); EXPECT_FALSE(errors.errors().empty()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); jobResult = jobResults[3]; ASSERT_TRUE(jobResult.job); EXPECT_FALSE(jobResult.measure); ASSERT_TRUE(jobResult.step.isWorkItem()); EXPECT_EQ(JobType(JobType::UserScript),jobResult.step.workItemType()); job = jobResult.job.get(); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); // now all four scripts are in same job EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); // now all four scripts are in same job errors = jobResult.errors().get(); EXPECT_EQ(OSResultValue(OSResultValue::Fail),errors.result); EXPECT_FALSE(errors.succeeded()); EXPECT_EQ(1u, errors.errors().size()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); jobResult = jobResults[4]; ASSERT_TRUE(jobResult.job); EXPECT_FALSE(jobResult.measure); ASSERT_TRUE(jobResult.step.isWorkItem()); EXPECT_EQ(JobType(JobType::ModelToIdf),jobResult.step.workItemType()); job = jobResult.job.get(); EXPECT_TRUE(job.outOfDate()); // never run EXPECT_FALSE(job.canceled()); EXPECT_FALSE(job.lastRun()); errors = jobResult.errors().get(); EXPECT_EQ(OSResultValue(OSResultValue::Fail),errors.result); EXPECT_FALSE(errors.succeeded()); EXPECT_TRUE(errors.errors().empty()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); EXPECT_EQ(OSResultValue(OSResultValue::Fail),treeErrors.result); EXPECT_FALSE(treeErrors.succeeded()); EXPECT_FALSE(treeErrors.errors().empty()); EXPECT_FALSE(treeErrors.warnings().empty()); EXPECT_FALSE(treeErrors.infos().empty()); } }
TEST_F(AnalysisDriverFixture,DataPersistence_DakotaErrors) { { // Create and populate project SimpleProject project = getCleanSimpleProject("DataPersistence_DakotaErrors"); Analysis analysis = project.analysis(); Problem problem = retrieveProblem(AnalysisDriverFixtureProblem::BuggyBCLMeasure, true, false); analysis.setProblem(problem); model::Model model = model::exampleModel(); openstudio::path p = toPath("./example.osm"); model.save(p,true); FileReference seedModel(p); project.setSeed(seedModel); DDACEAlgorithmOptions algOpts(DDACEAlgorithmType::lhs); // do not set samples so Dakota job will have errors DDACEAlgorithm alg(algOpts); analysis.setAlgorithm(alg); // Run analysis AnalysisRunOptions runOptions = standardRunOptions(project.projectDir()); project.analysisDriver().run(analysis,runOptions); project.analysisDriver().waitForFinished(); // Check Dakota job and error information ASSERT_TRUE(alg.job()); Job job = alg.job().get(); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); JobErrors errors = job.errors(); EXPECT_EQ(OSResultValue(OSResultValue::Fail),errors.result); EXPECT_FALSE(errors.succeeded()); EXPECT_FALSE(errors.errors().empty()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); } { // Re-open project SimpleProject project = getSimpleProject("DataPersistence_DakotaErrors"); Analysis analysis = project.analysis(); DDACEAlgorithm alg = analysis.algorithm()->cast<DDACEAlgorithm>(); // Verify job and error information still there ASSERT_TRUE(alg.job()); Job job = alg.job().get(); EXPECT_FALSE(job.running()); EXPECT_FALSE(job.outOfDate()); EXPECT_FALSE(job.canceled()); EXPECT_TRUE(job.lastRun()); JobErrors errors = job.errors(); EXPECT_EQ(OSResultValue(OSResultValue::Fail),errors.result); EXPECT_FALSE(errors.succeeded()); EXPECT_FALSE(errors.errors().empty()); EXPECT_TRUE(errors.warnings().empty()); EXPECT_TRUE(errors.infos().empty()); } }
TEST_F(CloudFixture,CloudAnalysisDriver_RunPrototypeProject) { { // open prototype project openstudio::path projectDir = vagrantServerPath().parent_path().parent_path() / toPath("prototype/pat/PATTest"); SimpleProjectOptions options; options.setLogLevel(Debug); SimpleProject project = openPATProject(projectDir,options).get(); // save as into new folder projectDir = AnalysisDriverTestLibrary::instance().outputDataDirectory() / toPath("CloudAnalysisDriver_RunPrototypeProject"); if (boost::filesystem::exists(projectDir)) { boost::filesystem::remove_all(projectDir); } OptionalSimpleProject temp = saveAs(project,projectDir); ASSERT_TRUE(temp); project = temp.get(); // run it CloudAnalysisDriver driver(provider->session(),project); driver.run(); EXPECT_TRUE(driver.lastRunSuccess()); // check data points BOOST_FOREACH(const DataPoint& dataPoint,project.analysis().dataPoints()) { EXPECT_TRUE(dataPoint.isComplete()); EXPECT_TRUE(dataPoint.runType() == DataPointRunType::CloudSlim); EXPECT_FALSE(dataPoint.outputAttributes().empty()); EXPECT_TRUE(dataPoint.directory().empty()); std::vector<WorkflowStepJob> jobsByStep = project.analysis().problem().getJobsByWorkflowStep(dataPoint); unsigned jobCount(0); unsigned messageCount(0); BOOST_FOREACH(const WorkflowStepJob& jobStep,jobsByStep) { if (jobStep.job) { ++jobCount; messageCount += jobStep.job.get().errors().errors().size(); messageCount += jobStep.job.get().errors().warnings().size(); messageCount += jobStep.job.get().errors().infos().size(); messageCount += jobStep.job.get().errors().initialConditions().size(); messageCount += jobStep.job.get().errors().finalConditions().size(); } } EXPECT_GT(jobCount,0u); EXPECT_GT(messageCount,0u); } } { // reopen prototype project and make sure data is still there openstudio::path projectDir = AnalysisDriverTestLibrary::instance().outputDataDirectory() / toPath("CloudAnalysisDriver_RunPrototypeProject"); SimpleProjectOptions options; options.setLogLevel(Debug); SimpleProject project = openPATProject(projectDir,options).get(); // check data points BOOST_FOREACH(const DataPoint& dataPoint,project.analysis().dataPoints()) { EXPECT_TRUE(dataPoint.isComplete()); EXPECT_TRUE(dataPoint.runType() == DataPointRunType::CloudSlim); EXPECT_FALSE(dataPoint.outputAttributes().empty()); EXPECT_TRUE(dataPoint.directory().empty()); std::vector<WorkflowStepJob> jobsByStep = project.analysis().problem().getJobsByWorkflowStep(dataPoint); unsigned jobCount(0); unsigned messageCount(0); BOOST_FOREACH(const WorkflowStepJob& jobStep,jobsByStep) { if (jobStep.job) { ++jobCount; messageCount += jobStep.job.get().errors().errors().size(); messageCount += jobStep.job.get().errors().warnings().size(); messageCount += jobStep.job.get().errors().infos().size(); messageCount += jobStep.job.get().errors().initialConditions().size(); messageCount += jobStep.job.get().errors().finalConditions().size(); } } EXPECT_GT(jobCount,0u); EXPECT_GT(messageCount,0u); } // now request detailed results CloudAnalysisDriver driver(provider->session(),project); DataPointVector dataPoints = project.analysis().dataPoints(); BOOST_FOREACH(DataPoint& dataPoint,dataPoints) { driver.requestDownloadDetailedResults(dataPoint); } driver.waitForFinished(); EXPECT_TRUE(driver.lastDownloadDetailedResultsSuccess()); // check outcome BOOST_FOREACH(const DataPoint& dataPoint,project.analysis().dataPoints()) { EXPECT_TRUE(dataPoint.isComplete()); EXPECT_TRUE(dataPoint.runType() == DataPointRunType::CloudDetailed); EXPECT_FALSE(dataPoint.outputAttributes().empty()); EXPECT_FALSE(dataPoint.directory().empty()); EXPECT_TRUE(dataPoint.model()); if (OptionalModel model = dataPoint.model()) { LOG(Debug,"DataPoint '" << dataPoint.name() << "' has " << model->numObjects() << " in its OpenStudio Model."); } } }