TEST_F(AnalysisDriverFixture, DDACE_MonteCarlo_MixedOsmIdf_ProjectDatabaseOpen) {
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("MixedOsmIdf",false,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::oas);
algOptions.setSamples(4);
Analysis analysis("DDACE Monte Carlo Sampling",
problem,
DDACEAlgorithm(algOptions),
seedModel);
// RUN ANALYSIS
{
analysis = Analysis("DDACE Monte Carlo Sampling - MixedOsmIdf",
problem,
DDACEAlgorithm(algOptions),
seedModel);
ProjectDatabase database = getCleanDatabase("DDACEMonteCarlo_MixedOsmIdf");
AnalysisDriver analysisDriver = AnalysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(5u,summary.nRows()); // 4 points
summary.save(analysisDriver.database().path().parent_path() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
}
}
{
project::OptionalProjectDatabase oDatabase = project::ProjectDatabase::open(toPath("AnalysisDriverFixtureData/DDACEMonteCarloMixedOsmIdf/DDACEMonteCarlo_MixedOsmIdf.osp"));
ASSERT_TRUE(oDatabase);
project::AnalysisRecordVector analysisRecords = project::AnalysisRecord::getAnalysisRecords(*oDatabase);
EXPECT_EQ(1u,analysisRecords.size());
if (!analysisRecords.empty()) {
EXPECT_NO_THROW(analysisRecords[0].analysis());
}
}
}
TEST_F(AnalysisDriverFixture, FSUDace_CVT_MixedOsmIdf) {
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("MixedOsmIdf",false,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
FSUDaceAlgorithmOptions algOptions(FSUDaceAlgorithmType::cvt);
algOptions.setSamples(3);
Analysis analysis("FSUDace CVT",
problem,
FSUDaceAlgorithm(algOptions),
seedModel);
// RUN ANALYSIS
analysis = Analysis("FSUDace CVT - MixedOsmIdf",
problem,
FSUDaceAlgorithm(algOptions),
seedModel);
ProjectDatabase database = getCleanDatabase("FSUDaceCVT_MixedOsmIdf");
AnalysisDriver analysisDriver = AnalysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
// EXPECT_EQ(10u,summary.nRows()); // 9 points
summary.save(analysisDriver.database().path().parent_path() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
}
}
TEST_F(AnalysisDriverFixture,RuntimeBehavior_StopDakotaAnalysis) {
// Tests for stopping time < 20s.
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("SimpleHistogramBinUQ",true,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
SamplingAlgorithmOptions algOptions;
algOptions.setSamples(100);
Analysis analysis("Stop Dakota Analysis",
problem,
SamplingAlgorithm(algOptions),
seedModel);
// RUN ANALYSIS
if (!dakotaExePath().empty()) {
ProjectDatabase database = getCleanDatabase("StopDakotaAnalysis");
AnalysisDriver analysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
StopWatcher watcher(analysisDriver);
watcher.watch(analysis.uuid());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
analysisDriver.waitForFinished();
EXPECT_FALSE(analysisDriver.isRunning());
EXPECT_GE(watcher.nComplete(),watcher.stopNum());
EXPECT_LE(watcher.stoppingTime(),openstudio::Time(0,0,0,20));
// check conditions afterward
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_FALSE(jobErrors->errors().empty());
EXPECT_FALSE(currentAnalysis.analysis().dataPoints().empty());
EXPECT_FALSE(currentAnalysis.analysis().algorithm()->isComplete());
EXPECT_FALSE(currentAnalysis.analysis().algorithm()->failed());
EXPECT_EQ(0u,analysisDriver.currentAnalyses().size());
}
}
TEST_F(AnalysisDriverFixture, DDACE_CentralComposite_Continuous) {
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("Continuous",true,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::central_composite);
Analysis analysis("DDACE Central Composite",
problem,
DDACEAlgorithm(algOptions),
seedModel);
// RUN ANALYSIS
if (!dakotaExePath().empty()) {
ProjectDatabase database = getCleanDatabase("DDACECentralComposite");
AnalysisDriver analysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
// output csv summary of data points
Table summary = currentAnalysis.analysis().summaryTable();
summary.save(analysisDriver.database().path().parent_path() / toPath("summary.csv"));
EXPECT_EQ(DDACEAlgorithmOptions::samplesForCentralComposite(problem),int(summary.nRows()-1));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
EXPECT_FALSE(dataPoint.responseValues().empty());
}
}
TEST_F(AnalysisDriverFixture, DDACE_LatinHypercube_UserScriptContinuous) {
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("UserScriptContinuous",true,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::lhs);
algOptions.setSamples(10);
// RUN ANALYSIS
Analysis analysis("DDACE Latin Hypercube Sampling - UserScriptContinuous",
problem,
DDACEAlgorithm(algOptions),
seedModel);
ProjectDatabase database = getCleanDatabase("DDACELatinHypercube_UserScriptContinuous");
AnalysisDriver analysisDriver = AnalysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(11u,summary.nRows());
summary.save(analysisDriver.database().path().parent_path() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
}
}
TEST_F(AnalysisDriverFixture, DDACE_MonteCarlo_Continuous) {
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("Continuous",true,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::random);
Analysis analysis("DDACE Monte Carlo Sampling",
problem,
DDACEAlgorithm(algOptions),
seedModel);
// RUN ANALYSIS
{
ProjectDatabase database = getCleanDatabase("DDACEMonteCarlo_Continuous_NoSamples");
AnalysisDriver analysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_FALSE(jobErrors->errors().empty()); // require specification of number of samples
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(1u,summary.nRows()); // no points
}
{
algOptions.setSamples(6);
analysis = Analysis("DDACE MonteCarlo Sampling",
problem,
DDACEAlgorithm(algOptions),
seedModel);
ProjectDatabase database = getCleanDatabase("DDACEMonteCarlo_Continuous");
AnalysisDriver analysisDriver = AnalysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
runOptions.setQueueSize(4);
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(7u,summary.nRows()); // 6 points
summary.save(analysisDriver.database().path().parent_path() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
// EXPECT_FALSE(dataPoint.responseValues().empty());
}
}
}
TEST_F(AnalysisDriverFixture,RuntimeBehavior_StopOpenStudioAnalysis) {
// Tests for stopping time < 20s.
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("IdfOnly",false,false);
// DEFINE SEED
FileReference seedModel(resourcesPath() / openstudio::toPath("energyplus/5ZoneAirCooled/in.idf"));
// CREATE ANALYSIS
DesignOfExperimentsOptions algOptions(DesignOfExperimentsType::FullFactorial);
Analysis analysis("Stop OpenStudio Analysis",
problem,
DesignOfExperiments(algOptions),
seedModel);
// RUN ANALYSIS
ProjectDatabase database = getCleanDatabase("StopOpenStudioAnalysis");
AnalysisDriver analysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
runOptions.setQueueSize(2);
StopWatcher watcher(analysisDriver);
watcher.watch(analysis.uuid());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_EQ(2,currentAnalysis.numQueuedJobs());
EXPECT_EQ(0,currentAnalysis.numQueuedDakotaJobs());
EXPECT_GE(currentAnalysis.totalNumJobsInOSIteration(),10);
EXPECT_EQ(0,currentAnalysis.numCompletedJobsInOSIteration());
analysisDriver.waitForFinished();
EXPECT_FALSE(analysisDriver.isRunning());
EXPECT_GE(watcher.nComplete(),watcher.stopNum());
EXPECT_LE(watcher.stoppingTime(),openstudio::Time(0,0,0,20));
// check conditions afterward
EXPECT_TRUE(currentAnalysis.numCompletedJobsInOSIteration() > 0);
EXPECT_TRUE(currentAnalysis.analysis().dataPointsToQueue().size() > 0u);
EXPECT_TRUE(currentAnalysis.analysis().dataPointsToQueue().size() <
currentAnalysis.analysis().dataPoints().size());
EXPECT_EQ(0u,analysisDriver.currentAnalyses().size());
}
TEST_F(AnalysisDriverFixture, DDACE_LatinHypercube_MixedOsmIdf_MoveProjectDatabase) {
openstudio::path oldDir, newDir;
{
// GET SIMPLE PROJECT
SimpleProject project = getCleanSimpleProject("DDACE_LatinHypercube_MixedOsmIdf");
Analysis analysis = project.analysis();
analysis.setName("DDACE Latin Hypercube Sampling - MixedOsmIdf");
// SET PROBLEM
Problem problem = retrieveProblem("MixedOsmIdf",false,false);
analysis.setProblem(problem);
// SET SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
analysis.setSeed(seedModel);
// SET ALGORITHM
DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::lhs);
algOptions.setSamples(12); // test reprinting results.out for copies of same point
DDACEAlgorithm algorithm(algOptions);
analysis.setAlgorithm(algorithm);
// RUN ANALYSIS
AnalysisDriver driver = project.analysisDriver();
AnalysisRunOptions runOptions = standardRunOptions(project.projectDir());
CurrentAnalysis currentAnalysis = driver.run(analysis,runOptions);
EXPECT_TRUE(driver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(driver.currentAnalyses().empty());
Table summary = analysis.summaryTable();
EXPECT_EQ(5u,summary.nRows()); // 4 points (all combinations)
summary.save(project.projectDir() / toPath("summary.csv"));
EXPECT_EQ(4u,analysis.dataPoints().size());
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
EXPECT_TRUE(dataPoint.workspace()); // should be able to load data from disk
}
oldDir = project.projectDir();
newDir = project.projectDir().parent_path() / toPath("DDACELatinHypercubeMixedOsmIdfCopy");
// Make copy of project
boost::filesystem::remove_all(newDir);
ASSERT_TRUE(project.saveAs(newDir));
}
// Blow away old project.
// TODO: Reinstate. This was failing on Windows and isn't absolutely necessary.
// try {
// boost::filesystem::remove_all(oldDir);
// }
// catch (std::exception& e) {
// EXPECT_TRUE(false) << "Boost filesystem was unable to delete the old folder, because " << e.what();
// }
// Open new project
SimpleProject project = getSimpleProject("DDACE_LatinHypercube_MixedOsmIdf_Copy");
EXPECT_TRUE(project.projectDir() == newDir);
EXPECT_EQ(toString(newDir),toString(project.projectDir()));
// After move, should be able to retrieve results.
EXPECT_FALSE(project.analysisIsLoaded());
Analysis analysis = project.analysis();
EXPECT_TRUE(project.analysisIsLoaded());
EXPECT_EQ(4u,analysis.dataPoints().size());
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
LOG(Debug,"Attempting to load workspace for data point at '" << dataPoint.directory() << "'.");
if (dataPoint.idfInputData()) {
LOG(Debug,"Says there should be input data at " << toString(dataPoint.idfInputData()->path()));
}
EXPECT_TRUE(dataPoint.workspace()); // should be able to load data from disk
if (!dataPoint.workspace()) {
LOG(Debug,"Unsuccessful.")
}
}
// Should be able to blow away results and run again
project.removeAllDataPoints();
EXPECT_EQ(0u,analysis.dataPoints().size());
EXPECT_FALSE(analysis.algorithm()->isComplete());
EXPECT_FALSE(analysis.algorithm()->failed());
EXPECT_EQ(-1,analysis.algorithm()->iter());
EXPECT_FALSE(analysis.algorithm()->cast<DakotaAlgorithm>().restartFileReference());
EXPECT_FALSE(analysis.algorithm()->cast<DakotaAlgorithm>().outFileReference());
AnalysisRunOptions runOptions = standardRunOptions(project.projectDir());
AnalysisDriver driver = project.analysisDriver();
CurrentAnalysis currentAnalysis = driver.run(analysis,runOptions);
EXPECT_TRUE(driver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(driver.currentAnalyses().empty());
Table summary = analysis.summaryTable();
EXPECT_EQ(5u,summary.nRows()); // 4 points (all combinations)
summary.save(project.projectDir() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
EXPECT_TRUE(dataPoint.workspace()); // should be able to load data from disk
}
}
TEST_F(AnalysisDriverFixture, DDACE_LatinHypercube_Continuous) {
{
// GET SIMPLE PROJECT
SimpleProject project = getCleanSimpleProject("DDACE_LatinHypercube_Continuous");
Analysis analysis = project.analysis();
// SET PROBLEM
Problem problem = retrieveProblem("Continuous",true,false);
analysis.setProblem(problem);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
analysis.setSeed(seedModel);
// CREATE ANALYSIS
DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::lhs);
DDACEAlgorithm algorithm(algOptions);
analysis.setAlgorithm(algorithm);
// RUN ANALYSIS
AnalysisDriver driver = project.analysisDriver();
AnalysisRunOptions runOptions = standardRunOptions(project.projectDir());
CurrentAnalysis currentAnalysis = driver.run(analysis,runOptions);
EXPECT_TRUE(driver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_FALSE(jobErrors->errors().empty()); // require specification of number of samples
EXPECT_TRUE(driver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(1u,summary.nRows()); // no points
project.clearAllResults();
algOptions.setSamples(4);
EXPECT_EQ(4,analysis.algorithm()->cast<DDACEAlgorithm>().ddaceAlgorithmOptions().samples());
currentAnalysis = driver.run(analysis,runOptions);
EXPECT_TRUE(driver.waitForFinished());
jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(driver.currentAnalyses().empty());
summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(5u,summary.nRows());
summary.save(project.projectDir() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
// EXPECT_FALSE(dataPoint.responseValues().empty());
}
ASSERT_TRUE(analysis.algorithm());
EXPECT_TRUE(analysis.algorithm()->isComplete());
EXPECT_FALSE(analysis.algorithm()->failed());
{
AnalysisRecord analysisRecord = project.analysisRecord();
Analysis analysisCopy = analysisRecord.analysis();
ASSERT_TRUE(analysisCopy.algorithm());
EXPECT_TRUE(analysisCopy.algorithm()->isComplete());
EXPECT_FALSE(analysisCopy.algorithm()->failed());
}
}
LOG(Info,"Restart from existing project.");
// Get existing project
SimpleProject project = getSimpleProject("DDACE_LatinHypercube_Continuous");
EXPECT_FALSE(project.analysisIsLoaded()); // make sure starting fresh
Analysis analysis = project.analysis();
EXPECT_FALSE(analysis.isDirty());
// Add custom data point
std::vector<QVariant> values;
values.push_back(0.0);
values.push_back(0.8);
values.push_back(int(0));
OptionalDataPoint dataPoint = analysis.problem().createDataPoint(values);
ASSERT_TRUE(dataPoint);
analysis.addDataPoint(*dataPoint);
EXPECT_EQ(1u,analysis.dataPointsToQueue().size());
ASSERT_TRUE(analysis.algorithm());
EXPECT_TRUE(analysis.algorithm()->isComplete());
EXPECT_FALSE(analysis.algorithm()->failed());
EXPECT_TRUE(analysis.isDirty());
EXPECT_FALSE(analysis.resultsAreInvalid());
EXPECT_FALSE(analysis.dataPointsAreInvalid());
// get last modified time of a file in a completed data point to make sure nothing is re-run
DataPointVector completePoints = analysis.completeDataPoints();
ASSERT_FALSE(completePoints.empty());
OptionalFileReference inputFileRef = completePoints[0].osmInputData();
ASSERT_TRUE(inputFileRef);
QFileInfo inputFileInfo(toQString(inputFileRef->path()));
QDateTime inputFileModifiedTestTime = inputFileInfo.lastModified();
EXPECT_EQ(1u,analysis.dataPointsToQueue().size());
AnalysisDriver driver = project.analysisDriver();
CurrentAnalysis currentAnalysis = driver.run(
analysis,
standardRunOptions(project.projectDir()));
EXPECT_TRUE(driver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
EXPECT_FALSE(jobErrors); // should not try to re-run DakotaAlgorithm
EXPECT_TRUE(driver.currentAnalyses().empty());
EXPECT_TRUE(analysis.dataPointsToQueue().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(6u,summary.nRows());
summary.save(project.projectDir() / toPath("summary_post_restart.csv"));
// RunManager should not re-run any data points
EXPECT_EQ(inputFileModifiedTestTime,inputFileInfo.lastModified());
}
TEST_F(AnalysisDriverFixture, DDACE_Grid_Continuous) {
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("Continuous",true,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::grid);
Analysis analysis("DDACE Grid Sampling",
problem,
DDACEAlgorithm(algOptions),
seedModel);
// RUN ANALYSIS
{
ProjectDatabase database = getCleanDatabase("DDACEGridSampling_NoSamples");
AnalysisDriver analysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
runOptions.setQueueSize(4);
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_FALSE(jobErrors->errors().empty()); // require specification of number of samples
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(1u,summary.nRows()); // no points
}
// ETH@20120120 At first tried just using the same database, analysisdriver, etc., but
// it did not go well. Restarting an initially failed Dakota analysis should be part of
// enabling Dakota restart more generally.
{
// algorithm rounds samples up to next one that fits n**(problem.numVariables())
algOptions.setSamples(6);
analysis = Analysis("DDACE Grid Sampling - Wrong Samples",
problem,
DDACEAlgorithm(algOptions),
seedModel);
ProjectDatabase database = getCleanDatabase("DDACEGridSampling_WrongSamples");
AnalysisDriver analysisDriver = AnalysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(10u,summary.nRows()); // 9 points
summary.save(analysisDriver.database().path().parent_path() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
// EXPECT_FALSE(dataPoint.responseValues().empty());
}
}
{
// algorithm rounds samples up to next one that fits n**(problem.numVariables())
algOptions.setSamplesForGrid(2,problem);
analysis = Analysis("DDACE Grid Sampling - Correct Samples",
problem,
DDACEAlgorithm(algOptions),
seedModel);
ProjectDatabase database = getCleanDatabase("DDACEGridSampling_CorrectSamples");
AnalysisDriver analysisDriver = AnalysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(5u,summary.nRows()); // 4 points
summary.save(analysisDriver.database().path().parent_path() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
// EXPECT_FALSE(dataPoint.responseValues().empty());
}
}
}
TEST_F(AnalysisDriverFixture, DDACE_OrthogonalArray_Continuous) {
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("Continuous",true,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
DDACEAlgorithmOptions algOptions(DDACEAlgorithmType::oas);
Analysis analysis("DDACE Orthogonal Array Sampling",
problem,
DDACEAlgorithm(algOptions),
seedModel);
// RUN ANALYSIS
{
ProjectDatabase database = getCleanDatabase("DDACEOrthogonalArray_NoSamples");
AnalysisDriver analysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_FALSE(jobErrors->errors().empty()); // require specification of number of samples
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(1u,summary.nRows()); // no points
}
{
algOptions.setSamples(6); // symbols = 0
analysis = Analysis("DDACE Orthogonal Array Sampling - Wrong Samples",
problem,
DDACEAlgorithm(algOptions),
seedModel);
ProjectDatabase database = getCleanDatabase("DDACEOrthogonalArray_WrongSamples");
AnalysisDriver analysisDriver = AnalysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(10u,summary.nRows()); // 9 points
summary.save(analysisDriver.database().path().parent_path() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
// EXPECT_FALSE(dataPoint.responseValues().empty());
}
}
{
bool ok = algOptions.setSamplesAndSymbolsForOrthogonalArray(6,1);
EXPECT_FALSE(ok);
ok = algOptions.setSamplesAndSymbolsForOrthogonalArray(2,2); // 2,3 not ok for no apparent reason
EXPECT_TRUE(ok);
ASSERT_TRUE(algOptions.symbols());
EXPECT_EQ(2,algOptions.symbols().get());
ASSERT_TRUE(algOptions.samples());
EXPECT_EQ(8,algOptions.samples().get());
analysis = Analysis("DDACE Orthogonal Array Sampling - Correct Samples",
problem,
DDACEAlgorithm(algOptions),
seedModel);
ProjectDatabase database = getCleanDatabase("DDACEOrthogonalArray_CorrectSamples");
AnalysisDriver analysisDriver = AnalysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_TRUE(analysisDriver.waitForFinished());
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_TRUE(jobErrors->errors().empty());
EXPECT_TRUE(analysisDriver.currentAnalyses().empty());
Table summary = currentAnalysis.analysis().summaryTable();
EXPECT_EQ(9u,summary.nRows());
summary.save(analysisDriver.database().path().parent_path() / toPath("summary.csv"));
BOOST_FOREACH(const DataPoint& dataPoint,analysis.dataPoints()) {
EXPECT_TRUE(dataPoint.isComplete());
EXPECT_FALSE(dataPoint.failed());
}
}
}
TEST_F(AnalysisDriverFixture,RuntimeBehavior_StopAndRestartDakotaAnalysis) {
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("SimpleHistogramBinUQ",true,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
SamplingAlgorithmOptions algOptions;
algOptions.setSamples(10);
Analysis analysis("Stop and Restart Dakota Analysis",
problem,
SamplingAlgorithm(algOptions),
seedModel);
// RUN ANALYSIS
ProjectDatabase database = getCleanDatabase("StopAndRestartDakotaAnalysis");
AnalysisDriver analysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
StopWatcher watcher(analysisDriver);
watcher.watch(analysis.uuid());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
analysisDriver.waitForFinished();
EXPECT_FALSE(analysisDriver.isRunning());
// check conditions afterward
boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
ASSERT_TRUE(jobErrors);
EXPECT_FALSE(jobErrors->errors().empty());
EXPECT_FALSE(currentAnalysis.analysis().dataPoints().empty());
EXPECT_FALSE(currentAnalysis.analysis().dataPointsToQueue().empty());
EXPECT_FALSE(currentAnalysis.analysis().completeDataPoints().empty());
EXPECT_FALSE(currentAnalysis.analysis().successfulDataPoints().empty());
EXPECT_TRUE(currentAnalysis.analysis().failedDataPoints().empty());
EXPECT_FALSE(currentAnalysis.analysis().algorithm()->isComplete());
EXPECT_FALSE(currentAnalysis.analysis().algorithm()->failed());
EXPECT_EQ(0u,analysisDriver.currentAnalyses().size());
LOG(Debug,"After initial stop, there are " << currentAnalysis.analysis().dataPoints().size()
<< " data points, of which " << currentAnalysis.analysis().completeDataPoints().size()
<< " are complete.");
// try to restart from database contents
analysis = AnalysisRecord::getAnalysisRecords(database)[0].analysis();
ASSERT_TRUE(analysis.algorithm());
EXPECT_FALSE(analysis.algorithm()->isComplete());
EXPECT_FALSE(analysis.algorithm()->failed());
currentAnalysis = analysisDriver.run(analysis,runOptions);
analysisDriver.waitForFinished();
EXPECT_EQ(10u,analysis.dataPoints().size());
EXPECT_EQ(0u,analysis.dataPointsToQueue().size());
EXPECT_EQ(10u,analysis.completeDataPoints().size());
EXPECT_EQ(10u,analysis.successfulDataPoints().size());
EXPECT_EQ(0u,analysis.failedDataPoints().size());
}
TEST_F(AnalysisDriverFixture,RuntimeBehavior_StopCustomAnalysis) {
// Tests for stopping time < 20s.
// RETRIEVE PROBLEM
Problem problem = retrieveProblem("UserScriptContinuous",true,false);
// DEFINE SEED
Model model = model::exampleModel();
openstudio::path p = toPath("./example.osm");
model.save(p,true);
FileReference seedModel(p);
// CREATE ANALYSIS
Analysis analysis("Stop Custom Analysis",
problem,
seedModel);
// generate 100 random points
boost::mt19937 mt;
typedef boost::uniform_real<> dist_type;
typedef boost::variate_generator<boost::mt19937&, dist_type > gen_type;
InputVariableVector variables = problem.variables();
ContinuousVariable cvar = variables[0].cast<ContinuousVariable>();
gen_type generator0(mt,dist_type(cvar.minimum().get(),cvar.maximum().get()));
cvar = variables[1].cast<ContinuousVariable>();
gen_type generator1(mt,dist_type(cvar.minimum().get(),cvar.maximum().get()));
cvar = variables[2].cast<ContinuousVariable>();
gen_type generator2(mt,dist_type(cvar.minimum().get(),cvar.maximum().get()));
for (int i = 0, n = 100; i < n; ++i) {
std::vector<QVariant> values;
double value = generator0();
values.push_back(value);
value = generator1();
values.push_back(value);
value = generator2();
values.push_back(value);
OptionalDataPoint dataPoint = problem.createDataPoint(values);
ASSERT_TRUE(dataPoint);
ASSERT_TRUE(analysis.addDataPoint(*dataPoint));
}
// RUN ANALYSIS
ProjectDatabase database = getCleanDatabase("StopCustomAnalysis");
AnalysisDriver analysisDriver(database);
AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
runOptions.setQueueSize(2);
StopWatcher watcher(analysisDriver);
watcher.watch(analysis.uuid());
CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
EXPECT_EQ(2,currentAnalysis.numQueuedJobs());
EXPECT_EQ(0,currentAnalysis.numQueuedDakotaJobs());
EXPECT_EQ(100,currentAnalysis.totalNumJobsInOSIteration());
EXPECT_EQ(0,currentAnalysis.numCompletedJobsInOSIteration());
analysisDriver.waitForFinished();
EXPECT_FALSE(analysisDriver.isRunning());
EXPECT_GE(watcher.nComplete(),watcher.stopNum());
EXPECT_LE(watcher.stoppingTime(),openstudio::Time(0,0,0,20));
// check conditions afterward
RunManager runManager = analysisDriver.database().runManager();
EXPECT_FALSE(runManager.workPending());
BOOST_FOREACH(const Job& job,runManager.getJobs()) {
EXPECT_FALSE(job.running());
EXPECT_FALSE(job.treeRunning());
}
EXPECT_TRUE(currentAnalysis.numCompletedJobsInOSIteration() > 0);
EXPECT_TRUE(currentAnalysis.analysis().dataPointsToQueue().size() > 0u);
EXPECT_TRUE(currentAnalysis.analysis().dataPointsToQueue().size() < 100u);
EXPECT_EQ(0u,analysisDriver.currentAnalyses().size());
}
