static void DoEvalBase(const ConfigParameters& config, IDataReader& reader)
{
DEVICEID_TYPE deviceId = DeviceFromConfig(config);
ConfigArray minibatchSize = config(L"minibatchSize", "40960");
size_t epochSize = config(L"epochSize", "0");
if (epochSize == 0)
{
epochSize = requestDataSize;
}
wstring modelPath = config(L"modelPath");
intargvector mbSize = minibatchSize;
int traceLevel = config(L"traceLevel", "0");
size_t numMBsToShowResult = config(L"numMBsToShowResult", "100");
ConfigArray evalNodeNames = config(L"evalNodeNames", "");
vector<wstring> evalNodeNamesVector;
for (int i = 0; i < evalNodeNames.size(); ++i)
{
evalNodeNamesVector.push_back(evalNodeNames[i]);
}
auto net = ComputationNetwork::CreateFromFile<ElemType>(deviceId, modelPath);
SimpleEvaluator<ElemType> eval(net, numMBsToShowResult, traceLevel);
eval.Evaluate(&reader, evalNodeNamesVector, mbSize[0], epochSize);
}
void ConfigList::load(std::istream &sin, string &line)
{
for (;;){
ConfigArray *el = create();
push_back(el);
el->load(sin, line);
if ((line == "[]") || (line == "")) return;
}
}
int wmain(int argc, wchar_t* argv[])
{
try
{
ConfigParameters config;
ConfigParameters::ParseCommandLine(argc, argv, config);
// get the command param set they want
wstring logpath = config("stderr", L"");
ConfigArray command = config("command", "train");
// dump config info
fprintf(stderr, "command: ");
for (int i = 0; i < command.size(); i++)
{
fprintf(stderr, "%s ", command[i].c_str());
}
// run commands
std::string type = config("precision", "float");
// accept old precision key for backward compatibility
if (config.Exists("type"))
type = config("type", "float");
fprintf(stderr, "\nprecision = %s\n", type.c_str());
if (type == "float")
DoCommand<float>(config);
else if (type == "double")
DoCommand<double>(config);
else
RuntimeError("invalid precision specified: %s", type.c_str());
}
catch (std::exception& err)
{
fprintf(stderr, "EXCEPTION occurred: %s", err.what());
Microsoft::MSR::CNTK::DebugUtil::PrintCallStack();
#ifdef _DEBUG
DebugBreak();
#endif
return -1;
}
catch (...)
{
fprintf(stderr, "Unknown ERROR occurred");
Microsoft::MSR::CNTK::DebugUtil::PrintCallStack();
#ifdef _DEBUG
DebugBreak();
#endif
return -1;
}
return 0;
}
static void DisableLegacyUsage(const ConfigParameters& TopLevelConfig, const ConfigArray& commands)
{
for (size_t i = 0; i < commands.size(); i++)
{
ConfigParameters cfgParameters(TopLevelConfig(commands[i]));
DisableLegacyTruncationSettings(TopLevelConfig, cfgParameters);
}
}
bool ConfigArray::operator == (const ConfigArray &a) const
{
list<ConfigValue*>::const_iterator it;
for (it = values.begin(); it != values.end(); it++){
const ConfigValue *v = a.getValue((*it)->m_name);
if (v == NULL) return false;
if (!(*v == *(*it))) return false;
}
return true;
}
ComputationNetworkPtr GetModelFromConfig(const ConfigRecordType& config, const wstring& outputNodeNamesConfig, vector<wstring>& outputNodeNamesVector)
{
DEVICEID_TYPE deviceId = DeviceFromConfig(config);
ConfigArray outputNodeNames = config(outputNodeNamesConfig.c_str(), ConfigArray(""));
ComputationNetworkPtr net;
// first try if a NetworkBuilder is present
function<ComputationNetworkPtr(DEVICEID_TYPE)> createNetworkFn;
bool gotIt = TryGetNetworkFactory<ConfigRecordType, ElemType>(config, createNetworkFn);
if (gotIt)
{
// We have several ways to create a network.
net = createNetworkFn(deviceId);
if (outputNodeNames.size() > 0)
{
net->InvalidateCompiledNetwork();
PatchOutputNodes(net, outputNodeNames, outputNodeNamesVector);
net->CompileNetwork();
// BUGBUG: This will generate double Validation output in the log
}
}
else // no NetworkBuilder given: load from 'modelPath'
{
wstring modelPath = config(L"modelPath");
// We don't use CreateFromFile() here since the user might specify OutputNodeNames in the config.
// By not compiling the network before patching, we avoid double log output for validation.
net = make_shared<ComputationNetwork>(deviceId);
net->SetTraceLevel(config(L"traceLevel", 0));
net->Read<ElemType>(modelPath);
if (outputNodeNames.size() > 0)
PatchOutputNodes(net, outputNodeNames, outputNodeNamesVector);
net->CompileNetwork();
}
return net;
}
// special temporary function to guard against a now invalid usage of "truncated" which exists in some IPG production setups
static void DisableLegacyTruncationSettings(const ConfigParameters& TopLevelConfig, const ConfigParameters& commandConfig)
{
if (TopLevelConfig.ExistsCurrent(L"Truncated"))
{
return;
}
// if any of the action has set a reader/SGD section and has different Truncated value for reader and SGD section
ConfigArray actions = commandConfig(L"action");
for (size_t i = 0; i < actions.size(); i++)
{
if (actions[i] == "train" || actions[i] == "trainRNN")
{
ConfigParameters sgd = ConfigParameters(commandConfig(L"SGD"));
ConfigParameters reader = ConfigParameters(commandConfig(L"reader"));
// reader and SGD sections are two must-have sections in train/trainRNN
if (reader.ExistsCurrent(L"Truncated") && !sgd.ExistsCurrent(L"Truncated"))
{
InvalidArgument("DisableLegacyUsage: setting Truncated only in reader section are not allowed. Please move Truncated=true/false to the top level section.");
}
}
}
}
void DoCrossValidate(const ConfigParameters& config)
{
// test
ConfigParameters readerConfig(config(L"reader"));
readerConfig.Insert("traceLevel", config(L"traceLevel", "0"));
DEVICEID_TYPE deviceId = DeviceFromConfig(config);
ConfigArray minibatchSize = config(L"minibatchSize", "40960");
size_t epochSize = config(L"epochSize", "0");
if (epochSize == 0)
{
epochSize = requestDataSize;
}
wstring modelPath = config(L"modelPath");
intargvector mbSize = minibatchSize;
ConfigArray cvIntervalConfig = config(L"crossValidationInterval");
intargvector cvInterval = cvIntervalConfig;
size_t sleepSecondsBetweenRuns = config(L"sleepTimeBetweenRuns", "0");
int traceLevel = config(L"traceLevel", 0);
size_t numMBsToShowResult = config(L"numMBsToShowResult", "100");
size_t firstMBsToShowResult = config(L"firstMBsToShowResult", "0");
size_t maxSamplesInRAM = config(L"maxSamplesInRAM", (size_t)SIZE_MAX);
size_t numSubminiBatches = config(L"numSubminibatches", (size_t)1);
bool enableDistributedMBReading = config(L"distributedMBReading", false);
ConfigArray evalNodeNames = config(L"evalNodeNames", "");
vector<wstring> evalNodeNamesVector;
for (int i = 0; i < evalNodeNames.size(); ++i)
{
evalNodeNamesVector.push_back(evalNodeNames[i]);
}
std::vector<std::vector<EpochCriterion>> cvErrorResults;
std::vector<std::wstring> cvModels;
DataReader cvDataReader(readerConfig);
bool finalModelEvaluated = false;
for (size_t i = cvInterval[0]; i <= cvInterval[2]; i += cvInterval[1])
{
wstring cvModelPath = msra::strfun::wstrprintf(L"%ls.%lld", modelPath.c_str(), i);
if (!fexists(cvModelPath))
{
fprintf(stderr, "Model %ls does not exist.\n", cvModelPath.c_str());
if (finalModelEvaluated || !fexists(modelPath))
continue; // file missing
else
{
cvModelPath = modelPath;
finalModelEvaluated = true;
}
}
cvModels.push_back(cvModelPath);
auto net = ComputationNetwork::CreateFromFile<ElemType>(deviceId, cvModelPath);
// BUGBUG: ^^ Should use GetModelFromConfig()
SimpleEvaluator<ElemType> eval(net, MPIWrapper::GetInstance(), enableDistributedMBReading, numMBsToShowResult,
firstMBsToShowResult, traceLevel, maxSamplesInRAM, numSubminiBatches);
fprintf(stderr, "Model %ls --> \n", cvModelPath.c_str());
auto evalErrors = eval.Evaluate(&cvDataReader, evalNodeNamesVector, mbSize[0], epochSize);
cvErrorResults.push_back(evalErrors);
::Sleep(1000 * sleepSecondsBetweenRuns);
}
// find best model
if (cvErrorResults.size() == 0)
LogicError("No model is evaluated.");
vector<double> minErrors;
vector<int> minErrIds;
vector<EpochCriterion> evalErrors = cvErrorResults[0];
for (int i = 0; i < evalErrors.size(); ++i)
{
minErrors.push_back(evalErrors[i].Average());
minErrIds.push_back(0);
}
for (int i = 0; i < cvErrorResults.size(); i++)
{
evalErrors = cvErrorResults[i];
for (int j = 0; j < evalErrors.size(); j++)
{
if (evalErrors[j].Average() < minErrors[j])
{
minErrors[j] = evalErrors[j].Average();
minErrIds[j] = i;
}
}
}
fprintf(stderr, "Best models:\n");
fprintf(stderr, "------------\n");
for (int i = 0; i < minErrors.size(); ++i)
fprintf(stderr, "Based on Err[%d]: Best model = %ls with min err %.8g\n", i, cvModels[minErrIds[i]].c_str(), minErrors[i]);
}
// called from wmain which is a wrapper that catches & repots Win32 exceptions
int wmainOldCNTKConfig(int argc, wchar_t* argv[])
{
std::string timestamp = TimeDateStamp();
PrintBanner(argc, argv, timestamp);
ConfigParameters config;
std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config); // get the command param set they want
int traceLevel = config(L"traceLevel", 0);
#ifndef CPUONLY
ConfigValue val = config("deviceId", "auto");
if (!EqualCI(val, "cpu") && !EqualCI(val, "auto"))
{
if (static_cast<int>(val) >= 0) // gpu (id >= 0)
{
CheckSupportForGpu(static_cast<int>(val)); // throws if gpu is not supported
}
}
#endif
if (config(L"timestamping", false))
ProgressTracing::SetTimestampingFlag();
if (config(L"forceDeterministicAlgorithms", false))
Globals::ForceDeterministicAlgorithms();
// get the command param set they want
wstring logpath = config(L"stderr", L"");
wstring doneFile = config(L"doneFile", L"");
ConfigArray command = config(L"command", "train");
// parallel training
// The top-level 'parallelTrain' is a bool, not to be confused with the parallelTrain block inside SGD.
shared_ptr<Microsoft::MSR::CNTK::MPIWrapper> mpi;
auto ensureMPIWrapperCleanup = MakeScopeExit(&MPIWrapper::DeleteInstance);
// when running under MPI with more than one node, use 'true' as the default value for parallelTrain,
// 'false' otherwise.
bool paralleltrain = config(L"parallelTrain", (MPIWrapper::GetTotalNumberOfMPINodes() > 1));
if (paralleltrain)
{
mpi = MPIWrapper::GetInstance(true /*create*/);
}
g_shareNodeValueMatrices = config(L"shareNodeValueMatrices", false);
TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0));
if (logpath != L"")
{
#if 1 // keep the ability to do it how it was done before 1.8; delete if noone needs it anymore
let useOldWay = ProgressTracing::GetTimestampingFlag(); // enable it when running in our server farm
if (useOldWay)
{
for (int i = 0; i < command.size(); i++) // append all 'command' entries
{
logpath += L"_";
logpath += (wstring)command[i];
}
logpath += L".log"; // append .log
}
if (paralleltrain && useOldWay)
{
std::wostringstream oss;
oss << mpi->CurrentNodeRank();
logpath += L"rank" + oss.str();
}
else
#endif
// for MPI workers except main, append .rankN
if (paralleltrain && mpi->CurrentNodeRank() != 0)
logpath += msra::strfun::wstrprintf(L".rank%d", mpi->CurrentNodeRank());
RedirectStdErr(logpath);
if (traceLevel == 0)
PrintBanner(argc, argv, timestamp); // repeat simple banner into log file
}
// full config info
if (traceLevel > 0)
{
PrintBuiltInfo();
PrintGpuInfo();
}
#ifdef _DEBUG
if (traceLevel > 0)
{
// This simply merges all the different config parameters specified (eg, via config files or via command line directly),
// and prints it.
fprintf(stderr, "\nConfiguration, Raw:\n\n");
LOGPRINTF(stderr, "%s\n", rawConfigString.c_str());
// Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overridden at command line),
// All of these assignments will appear, even though only the last assignment matters.
fprintf(stderr, "\nConfiguration After Variable Resolution:\n\n");
LOGPRINTF(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str());
}
#endif
SetMathLibTraceLevel(traceLevel);
// This outputs the final value each variable/parameter is assigned to in config (so if a parameter is set multiple times, only the last
// value it is set to will appear).
if (traceLevel > 0)
{
fprintf(stderr, "\nConfiguration After Processing and Variable Resolution:\n\n");
config.dumpWithResolvedVariables();
LOGPRINTF(stderr, "Commands:");
for (int i = 0; i < command.size(); i++)
fprintf(stderr, " %s", command[i].c_str());
fprintf(stderr, "\n");
}
// run commands
std::string type = config(L"precision", "float");
// accept old precision key for backward compatibility
if (config.Exists("type"))
InvalidArgument("CNTK: Use of 'type' parameter is deprecated, it is called 'precision' now.");
if (traceLevel > 0)
{
LOGPRINTF(stderr, "precision = \"%s\"\n", type.c_str());
}
if (type == "float")
DoCommands<float>(config, mpi);
else if (type == "double")
DoCommands<double>(config, mpi);
else
RuntimeError("CNTK: Invalid precision string: \"%s\", must be \"float\" or \"double\"", type.c_str());
// if completed then write a doneFile if requested
if (!doneFile.empty())
{
FILE* fp = fopenOrDie(doneFile.c_str(), L"w");
fprintf(fp, "Successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str());
fcloseOrDie(fp);
}
if (ProgressTracing::GetTimestampingFlag())
{
LOGPRINTF(stderr, "__COMPLETED__\n"); // running in server environment which expects this string
}
else
fprintf(stderr, "COMPLETED.\n");
fflush(stderr);
return EXIT_SUCCESS;
}
void DoCommands(const ConfigParameters& config, const shared_ptr<MPIWrapper>& mpi)
{
ConfigArray command = config(L"command", "train");
if (Globals::ShouldForceDeterministicAlgorithms())
ForceDeterministicAlgorithmsOnCPU();
else
{
// Setting specified number of threads.
int numCPUThreads = config(L"numCPUThreads", "0");
numCPUThreads = CPUMatrix<ElemType>::SetNumThreads(numCPUThreads);
if (numCPUThreads > 0)
{
LOGPRINTF(stderr, "Using %d CPU threads.\n", numCPUThreads);
}
}
bool progressTracing = config(L"progressTracing", false);
// temporary hack to prevent users from failing due to a small breaking change related to the "truncated" flag (will be redone bigger and better some day)
DisableLegacyUsage(config, command);
// summarize command info upfront in the log and stdout
size_t fullTotalMaxEpochs = 0;
for (int i = 0; i < command.size(); i++)
{
// get the configuration parameters that match the command
ConfigParameters commandParams(config(command[i]));
ConfigArray action = commandParams("action", "train");
// determine the action to perform, and do it
for (int j = 0; j < action.size(); j++)
{
if (action[j] == "train" || action[j] == "trainRNN")
{
wstring modelPath = commandParams("modelPath");
size_t maxEpochs = GetMaxEpochs(commandParams);
if (progressTracing)
{
LOGPRINTF(stderr, "CNTKModelPath: %ls\n", modelPath.c_str());
LOGPRINTF(stderr, "CNTKCommandTrainInfo: %s : %d\n", command[i].c_str(), (int)maxEpochs);
}
fullTotalMaxEpochs += maxEpochs;
}
}
}
if (progressTracing)
{
LOGPRINTF(stderr, "CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : %d\n", (int)fullTotalMaxEpochs);
}
// set up progress tracing for compute cluster management
if (progressTracing && (!mpi || mpi->IsMainNode()))
{
ProgressTracing::SetTracingFlag();
ProgressTracing::TraceTotalNumberOfSteps(fullTotalMaxEpochs); // enable tracing, using this as the total number of epochs
}
size_t fullEpochsOffset = 0;
// execute the commands
for (int i = 0; i < command.size(); i++)
{
// get the configuration parameters that match the command
const string thisCommand = command[i];
ConfigParameters commandParams(config(thisCommand));
ConfigArray action = commandParams("action", "train");
int traceLevel = commandParams("traceLevel", "0");
if (progressTracing && ((mpi == nullptr) || mpi->IsMainNode()))
{
ProgressTracing::SetStepOffset(fullEpochsOffset); // this is the epoch number that SGD will log relative to
}
// determine the action to perform, and do it
for (int j = 0; j < action.size(); j++)
{
const string thisAction = action[j];
// print a banner to visually separate each action in the log
const char* delim = "##############################################################################";
string showActionAs = thisCommand + " command (" + thisAction + " action)";
fprintf(stderr, "\n");
LOGPRINTF(stderr, "%s\n", delim);
LOGPRINTF(stderr, "#%*s#\n", (int)(strlen(delim) - 2), "");
LOGPRINTF(stderr, "# %s%*s #\n", showActionAs.c_str(), (int)(strlen(delim) - showActionAs.size() - 4), "");
LOGPRINTF(stderr, "#%*s#\n", (int)(strlen(delim) - 2), "");
LOGPRINTF(stderr, "%s\n\n", delim);
if ((mpi == nullptr) || (commandstoRunOnAllRanks.find(thisAction) != commandstoRunOnAllRanks.end()) || mpi->IsMainNode())
{
if (thisAction == "train" || thisAction == "trainRNN")
{
if (progressTracing)
{
LOGPRINTF(stderr, "CNTKCommandTrainBegin: %s\n", command[i].c_str());
}
DoTrain<ConfigParameters, ElemType>(commandParams);
if (progressTracing)
{
LOGPRINTF(stderr, "CNTKCommandTrainEnd: %s\n", command[i].c_str());
}
fullEpochsOffset += GetMaxEpochs(commandParams);
}
// TODO: Choose a clearer name.
else if (thisAction == "pbn")
{
DoEvalBN<ElemType>(commandParams);
}
else if (thisAction == "adapt")
{
DoAdapt<ElemType>(commandParams);
}
else if (thisAction == "test" || thisAction == "eval")
{
DoEval<ElemType>(commandParams);
}
else if (thisAction == "edit")
{
DoEdit<ElemType>(commandParams);
}
else if (thisAction == "cv")
{
DoCrossValidate<ElemType>(commandParams);
}
else if (thisAction == "write")
{
DoWriteOutput<ElemType>(commandParams);
}
else if (thisAction == "devtest")
{
TestCn<ElemType>(config); // for "devtest" action pass the root config instead
}
else if (thisAction == "dumpNodes" /*deprecated:*/ || thisAction == "dumpNode" || thisAction == "dumpnode")
{
DoDumpNodes<ElemType>(commandParams);
}
else if (thisAction == "convertdbn")
{
DoConvertFromDbn<ElemType>(commandParams);
}
else if (thisAction == "exportdbn")
{
DoExportToDbn<ElemType>(commandParams);
}
else if (thisAction == "createLabelMap")
{
DoCreateLabelMap<ElemType>(commandParams);
}
else if (thisAction == "writeWordAndClass")
{
DoWriteWordAndClassInfo<ElemType>(commandParams);
}
else if (thisAction == "plot")
{
DoTopologyPlot<ElemType>(commandParams);
}
else if (thisAction == "SVD")
{
DoParameterSVD<ElemType>(commandParams);
}
else
{
RuntimeError("unknown action: %s in command set: %s", thisAction.c_str(), command[i].c_str());
}
}
fprintf(stderr, "\n");
if (traceLevel > 0)
{
LOGPRINTF(stderr, "Action \"%s\" complete.\n\n", thisAction.c_str());
}
NDLScript<ElemType> ndlScript;
ndlScript.ClearGlobal(); // clear global macros between commands
// Synchronize all ranks before proceeding to next action/command
if (mpi)
mpi->WaitAll();
}
}
}
void DoWriteOutput(const ConfigParameters& config)
{
ConfigParameters readerConfig(config(L"reader"));
readerConfig.Insert("traceLevel", config(L"traceLevel", "0"));
readerConfig.Insert("randomize", "None"); // we don't want randomization when output results
DataReader testDataReader(readerConfig);
DEVICEID_TYPE deviceId = DeviceFromConfig(config);
ConfigArray minibatchSize = config(L"minibatchSize", "2048");
wstring modelPath = config(L"modelPath");
intargvector mbSize = minibatchSize;
size_t epochSize = config(L"epochSize", "0");
if (epochSize == 0)
{
epochSize = requestDataSize;
}
ConfigArray outputNodeNames = config(L"outputNodeNames", "");
vector<wstring> outputNodeNamesVector;
// Note this is required since the user might specify OutputNodeNames in the config, so don't use CreateFromFile,
// instead we build the network ourselves.
auto net = make_shared<ComputationNetwork>(deviceId);
net->Read<ElemType>(modelPath);
if (outputNodeNames.size() > 0)
{
net->OutputNodes().clear();
for (int i = 0; i < outputNodeNames.size(); ++i)
{
outputNodeNamesVector.push_back(outputNodeNames[i]);
net->OutputNodes().emplace_back(net->GetNodeFromName(outputNodeNames[i]));
}
}
net->CompileNetwork();
SimpleOutputWriter<ElemType> writer(net, 1);
if (config.Exists("writer"))
{
ConfigParameters writerConfig(config(L"writer"));
bool bWriterUnittest = writerConfig(L"unittest", "false");
DataWriter testDataWriter(writerConfig);
writer.WriteOutput(testDataReader, mbSize[0], testDataWriter, outputNodeNamesVector, epochSize, bWriterUnittest);
}
else if (config.Exists("outputPath"))
{
wstring outputPath = config(L"outputPath");
// gather additional formatting options
typename decltype(writer)::WriteFormattingOptions formattingOptions;
if (config.Exists("format"))
{
ConfigParameters formatConfig(config(L"format"));
if (formatConfig.ExistsCurrent("type")) // do not inherit 'type' from outer block
{
string type = formatConfig(L"type");
if (type == "real") formattingOptions.isCategoryLabel = false;
else if (type == "category") formattingOptions.isCategoryLabel = true;
else InvalidArgument("write: type must be 'real' or 'category'");
if (formattingOptions.isCategoryLabel)
formattingOptions.labelMappingFile = (wstring)formatConfig(L"labelMappingFile", L"");
}
formattingOptions.transpose = formatConfig(L"transpose", formattingOptions.transpose);
formattingOptions.prologue = formatConfig(L"prologue", formattingOptions.prologue);
formattingOptions.epilogue = formatConfig(L"epilogue", formattingOptions.epilogue);
formattingOptions.sequenceSeparator = formatConfig(L"sequenceSeparator", formattingOptions.sequenceSeparator);
formattingOptions.sequencePrologue = formatConfig(L"sequencePrologue", formattingOptions.sequencePrologue);
formattingOptions.sequenceEpilogue = formatConfig(L"sequenceEpilogue", formattingOptions.sequenceEpilogue);
formattingOptions.elementSeparator = formatConfig(L"elementSeparator", formattingOptions.elementSeparator);
formattingOptions.sampleSeparator = formatConfig(L"sampleSeparator", formattingOptions.sampleSeparator);
formattingOptions.precisionFormat = formatConfig(L"precisionFormat", formattingOptions.precisionFormat);
}
writer.WriteOutput(testDataReader, mbSize[0], outputPath, outputNodeNamesVector, formattingOptions, epochSize);
}
else
InvalidArgument("write command: You must specify either 'writer'or 'outputPath'");
}
int wmainOldCNTKConfig(int argc, wchar_t* argv[]) // called from wmain which is a wrapper that catches & repots Win32 exceptions
{
ConfigParameters config;
std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config);
// get the command param set they want
wstring logpath = config(L"stderr", L"");
// [1/26/2015 erw, add done file so that it can be used on HPC]
wstring DoneFile = config(L"DoneFile", L"");
ConfigArray command = config(L"command", "train");
// paralleltrain training
g_mpi = nullptr;
bool paralleltrain = config(L"parallelTrain", "false");
if (paralleltrain)
{
g_mpi = new MPIWrapper();
}
g_shareNodeValueMatrices = config(L"shareNodeValueMatrices", false);
TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0));
if (logpath != L"")
{
for (int i = 0; i < command.size(); i++)
{
logpath += L"_";
logpath += (wstring) command[i];
}
logpath += L".log";
if (paralleltrain)
{
std::wostringstream oss;
oss << g_mpi->CurrentNodeRank();
logpath += L"rank" + oss.str();
}
RedirectStdErr(logpath);
}
PrintBuiltInfo(); // this one goes to log file
std::string timestamp = TimeDateStamp();
// dump config info
fprintf(stderr, "running on %s at %s\n", GetHostName().c_str(), timestamp.c_str());
fprintf(stderr, "command line: \n");
for (int i = 0; i < argc; i++)
{
fprintf(stderr, "%s ", WCharToString(argv[i]).c_str());
}
// This simply merges all the different config parameters specified (eg, via config files or via command line directly),
// and prints it.
fprintf(stderr, "\n\n>>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>>\n");
fprintf(stderr, "%s\n", rawConfigString.c_str());
fprintf(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<<\n");
// Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overriden at command line),
// All of these assignments will appear, even though only the last assignment matters.
fprintf(stderr, "\n>>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n");
fprintf(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str());
fprintf(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n");
// This outputs the final value each variable/parameter is assigned to in config (so if a parameter is set multiple times, only the last
// value it is set to will appear).
fprintf(stderr, "\n>>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n");
config.dumpWithResolvedVariables();
fprintf(stderr, "<<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n");
fprintf(stderr, "command: ");
for (int i = 0; i < command.size(); i++)
{
fprintf(stderr, "%s ", command[i].c_str());
}
// run commands
std::string type = config(L"precision", "float");
// accept old precision key for backward compatibility
if (config.Exists("type"))
{
type = config(L"type", "float");
}
fprintf(stderr, "\nprecision = %s\n", type.c_str());
if (type == "float")
{
DoCommands<float>(config);
}
else if (type == "double")
{
DoCommands<double>(config);
}
else
{
RuntimeError("invalid precision specified: %s", type.c_str());
}
// still here , write a DoneFile if necessary
if (!DoneFile.empty())
{
FILE* fp = fopenOrDie(DoneFile.c_str(), L"w");
fprintf(fp, "successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str());
fcloseOrDie(fp);
}
fprintf(stderr, "COMPLETED\n"), fflush(stderr);
delete g_mpi;
return EXIT_SUCCESS;
}
void DoCommands(const ConfigParameters& config)
{
ConfigArray command = config(L"command", "train");
int numCPUThreads = config(L"numCPUThreads", "0");
numCPUThreads = CPUMatrix<ElemType>::SetNumThreads(numCPUThreads);
if (numCPUThreads > 0)
{
std::cerr << "Using " << numCPUThreads << " CPU threads" << endl;
}
bool progressTracing = config(L"progressTracing", false);
// temporary hack to prevent users from failling for a small breaking change related to the "truncated" flag (will be redone bigger and better some day)
DisableLegacyUsage(config, command);
// summarize command info upfront in the log and stdout
size_t fullTotalMaxEpochs = 0;
for (int i = 0; i < command.size(); i++)
{
// get the configuration parameters that match the command
ConfigParameters commandParams(config(command[i]));
ConfigArray action = commandParams("action", "train");
// determine the action to perform, and do it
for (int j = 0; j < action.size(); j++)
{
if (action[j] == "train" || action[j] == "trainRNN")
{
wstring modelPath = commandParams("modelPath");
std::wcerr << "CNTKModelPath: " << modelPath << endl;
size_t maxEpochs = GetMaxEpochs(commandParams);
std::cerr << "CNTKCommandTrainInfo: " + command[i] << " : " << maxEpochs << endl;
fullTotalMaxEpochs += maxEpochs;
}
}
}
std::cerr << "CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : " << fullTotalMaxEpochs << endl;
// set up progress tracing for compute cluster management
if (progressTracing && ((g_mpi == nullptr) || g_mpi->IsMainNode()))
{
ProgressTracing::TraceTotalNumberOfSteps(fullTotalMaxEpochs); // enable tracing, using this as the total number of epochs
}
size_t fullEpochsOffset = 0;
// execute the commands
for (int i = 0; i < command.size(); i++)
{
// get the configuration parameters that match the command
ConfigParameters commandParams(config(command[i]));
ConfigArray action = commandParams("action", "train");
if (progressTracing && ((g_mpi == nullptr) || g_mpi->IsMainNode()))
{
ProgressTracing::SetStepOffset(fullEpochsOffset); // this is the epoch number that SGD will log relative to
}
// determine the action to perform, and do it
for (int j = 0; j < action.size(); j++)
{
if (action[j] == "train" || action[j] == "trainRNN")
{
std::cerr << "CNTKCommandTrainBegin: " + command[i] << endl;
DoTrain<ConfigParameters, ElemType>(commandParams);
std::cerr << "CNTKCommandTrainEnd: " + command[i] << endl;
fullEpochsOffset += GetMaxEpochs(commandParams);
}
else if (action[j] == "adapt")
{
DoAdapt<ElemType>(commandParams);
}
else if (action[j] == "test" || action[j] == "eval")
{
DoEval<ElemType>(commandParams);
}
else if (action[j] == "edit")
{
DoEdit<ElemType>(commandParams);
}
else if (action[j] == "cv")
{
DoCrossValidate<ElemType>(commandParams);
}
else if (action[j] == "write")
{
DoWriteOutput<ElemType>(commandParams);
}
else if (action[j] == "devtest")
{
TestCn<ElemType>(config); // for "devtest" action pass the root config instead
}
else if (action[j] == "dumpnode")
{
DumpNodeInfo<ElemType>(commandParams);
}
else if (action[j] == "convertdbn")
{
DoConvertFromDbn<ElemType>(commandParams);
}
else if (action[j] == "createLabelMap")
{
DoCreateLabelMap<ElemType>(commandParams);
}
else if (action[j] == "writeWordAndClass")
{
DoWriteWordAndClassInfo<ElemType>(commandParams);
}
else if (action[j] == "plot")
{
DoTopologyPlot<ElemType>(commandParams);
}
else if (action[j] == "SVD")
{
DoParameterSVD<ElemType>(commandParams);
}
else
{
RuntimeError("unknown action: %s in command set: %s", action[j].c_str(), command[i].c_str());
}
NDLScript<ElemType> ndlScript;
ndlScript.ClearGlobal(); // clear global macros between commands
}
}
}
// ---------------------------------------------------------------------------
// main() for old CNTK config language
// ---------------------------------------------------------------------------
// called from wmain which is a wrapper that catches & repots Win32 exceptions
int wmainOldCNTKConfig(int argc, wchar_t* argv[])
{
ConfigParameters config;
std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config); // get the command param set they want
bool timestamping = config(L"timestamping", false);
if (timestamping)
{
ProgressTracing::SetTimestampingFlag();
}
// get the command param set they want
wstring logpath = config(L"stderr", L"");
// [1/26/2015 erw, add done file so that it can be used on HPC]
wstring DoneFile = config(L"DoneFile", L"");
ConfigArray command = config(L"command", "train");
// paralleltrain training
shared_ptr<Microsoft::MSR::CNTK::MPIWrapper> mpi;
bool paralleltrain = config(L"parallelTrain", "false");
if (paralleltrain)
mpi = MPIWrapper::GetInstance(true /*create*/);
g_shareNodeValueMatrices = config(L"shareNodeValueMatrices", false);
TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0));
if (logpath != L"")
{
for (int i = 0; i < command.size(); i++)
{
logpath += L"_";
logpath += (wstring) command[i];
}
logpath += L".log";
if (paralleltrain)
{
std::wostringstream oss;
oss << mpi->CurrentNodeRank();
logpath += L"rank" + oss.str();
}
RedirectStdErr(logpath);
}
PrintBuiltInfo(); // this one goes to log file
std::string timestamp = TimeDateStamp();
// dump config info
fprintf(stderr, "\n");
LOGPRINTF(stderr, "Running on %s at %s\n", GetHostName().c_str(), timestamp.c_str());
LOGPRINTF(stderr, "Command line: \n");
for (int i = 0; i < argc; i++)
fprintf(stderr, "%*s%ls", i > 0 ? 2 : 0, "", argv[i]); // use 2 spaces for better visual separability
fprintf(stderr, "\n\n");
#if 1 //def _DEBUG
// This simply merges all the different config parameters specified (eg, via config files or via command line directly),
// and prints it.
fprintf(stderr, "\n\n");
LOGPRINTF(stderr, ">>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>>\n");
LOGPRINTF(stderr, "%s\n", rawConfigString.c_str());
LOGPRINTF(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<<\n");
// Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overridden at command line),
// All of these assignments will appear, even though only the last assignment matters.
fprintf(stderr, "\n");
LOGPRINTF(stderr, ">>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n");
LOGPRINTF(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str());
LOGPRINTF(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n");
// This outputs the final value each variable/parameter is assigned to in config (so if a parameter is set multiple times, only the last
// value it is set to will appear).
fprintf(stderr, "\n");
LOGPRINTF(stderr, ">>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n");
config.dumpWithResolvedVariables();
LOGPRINTF(stderr, "<<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n");
#endif
LOGPRINTF(stderr, "Commands:");
for (int i = 0; i < command.size(); i++)
fprintf(stderr, " %s", command[i].c_str());
fprintf(stderr, "\n");
// run commands
std::string type = config(L"precision", "float");
// accept old precision key for backward compatibility
if (config.Exists("type"))
InvalidArgument("CNTK: Use of 'type' parameter is deprecated, it is called 'precision' now.");
LOGPRINTF(stderr, "Precision = \"%s\"\n", type.c_str());
if (type == "float")
DoCommands<float>(config, mpi);
else if (type == "double")
DoCommands<double>(config, mpi);
else
RuntimeError("CNTK: Invalid precision string: \"%s\", must be \"float\" or \"double\"", type.c_str());
// if completed then write a DoneFile if requested
if (!DoneFile.empty())
{
FILE* fp = fopenOrDie(DoneFile.c_str(), L"w");
fprintf(fp, "successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str());
fcloseOrDie(fp);
}
// TODO: Change back to COMPLETED (no underscores)
LOGPRINTF(stderr, "__COMPLETED__\n");
fflush(stderr);
MPIWrapper::DeleteInstance();
return EXIT_SUCCESS;
}
void TestConfiguration(const ConfigParameters& configBase)
{
ConfigParameters configMacros = configBase("macroExample");
for (auto iterMacro = configMacros.begin(); iterMacro != configMacros.end(); iterMacro++)
{
std::map<std::string, ConfigValue> paramsMap;
ConfigParameters configCN = iterMacro->second;
if (configCN.Exists("parameters"))
{
ConfigArray params = configCN("parameters");
for (int i = 0; i < params.size(); ++i)
paramsMap[params[i]] = ConfigValue("uninitialized");
}
ConfigParameters configNodes = configCN("NodeList");
for (auto iter = configNodes.begin();
iter != configNodes.end(); iter++)
{
std::wstring nodeName;
nodeName = msra::strfun::utf16(iter->first);
ConfigArray configNode = iter->second;
std::string opName = configNode[0];
if (IsParameter(paramsMap, opName))
{
;
}
if (opName == "InputValue" && configNode.size() >= 2)
{
size_t rows = 0;
if (!IsParameter(paramsMap, configNode[1]))
rows = configNode[1];
}
else if (opName == "LearnableParameter" && configNode.size() >= 3)
{
size_t rows = 0;
if (!IsParameter(paramsMap, configNode[1]))
rows = configNode[1];
size_t cols = 0;
if (!IsParameter(paramsMap, configNode[2]))
cols = configNode[2];
bool learningRateMultiplier = 0;
bool init = false;
ConfigArray initData;
// look for optional parameters
for (int i = 3; i < configNode.size(); ++i)
{
bool needsGradient = false;
ConfigParameters configParam = configNode[i];
if (configParam.Exists("learningRateMultiplier")) // TODO: should this be a test for 'true' rather than Exists()?
needsGradient = (float)configParam("learningRateMultiplier") > 0? true : false;
else if (configParam.Exists("init"))
{
init = true;
initData = configParam["init"];
}
}
// if initializing, do so now
if (init)
{
bool uniform = true;
ElemType initValueScale = 1;
size_t inputSize = cols;
if (initData.size() > 0)
initValueScale = initData[0];
if (initData.size() > 1)
uniform = EqualCI(initData[1], "uniform");
}
}
}
// now link up all the nodes
configNodes = configCN("Relation");
for (auto iter = configNodes.begin(); iter != configNodes.end(); iter++)
{
std::wstring nodeName = msra::strfun::utf16(iter->first);
ConfigArray configNode = iter->second;
int numChildren = (int) configNode.size();
for (int i = 0; i < numChildren; ++i)
{
std::wstring nodeName = configNode[i];
}
}
ConfigParameters configRoots = configCN("RootNodes");
ConfigArray configNode = configRoots("FeatureNodes");
for (size_t i = 0; i < configNode.size(); i++)
{
std::wstring nodeName = configNode[i];
}
if (configRoots.Exists("LabelNodes"))
{
configNode = configRoots("LabelNodes");
for (size_t i = 0; i < configNode.size(); i++)
{
std::wstring nodeName = configNode[i];
}
}
if (configRoots.Exists("CriterionNodes"))
{
configNode = configRoots("CriterionNodes");
for (size_t i = 0; i < configNode.size(); i++)
{
std::wstring nodeName = configNode[i];
}
}
if (configRoots.Exists("CriteriaNodes")) // legacy
{
configNode = configRoots("CriteriaNodes");
for (size_t i = 0; i < configNode.size(); i++)
{
std::wstring nodeName = configNode[i];
}
}
if (configRoots.Exists("NodesReqMultiSeqHandling"))
{
configNode = configRoots("NodesReqMultiSeqHandling");
for (size_t i = 0; i < configNode.size(); i++)
{
std::wstring nodeName = configNode[i];
}
fprintf(stderr, "WARNING: 'NodesReqMultiSeqHandling' flag is defunct\n");
}
if (configRoots.Exists("EvalNodes"))
{
configNode = configRoots("EvalNodes");
for (size_t i = 0; i < configNode.size(); i++)
{
std::wstring nodeName = configNode[i];
}
}
if (configRoots.Exists("OutputNodes"))
{
configNode = configRoots("OutputNodes");
for (size_t i = 0; i < configNode.size(); i++)
{
std::wstring nodeName = configNode[i];
}
}
}
}
Section* BinaryWriter<ElemType>::CreateSection(const ConfigParameters& config, Section* parentSection, size_t p_records, size_t p_windowSize)
{
// first check if we need to open a new section file
std::vector<std::wstring> sections;
// determine the element size, default to ElemType size
size_t elementSize = sizeof(ElemType);
if (config.ExistsCurrent(L"elementSize"))
{
elementSize = config(L"elementSize");
}
// get the number of records we should expect (max)
// if defined in previous levels same number will be used
size_t records = p_records;
if (config.ExistsCurrent(L"wrecords"))
{
records = config(L"wrecords");
}
if (records == 0)
{
InvalidArgument("Required config variable 'wrecords' missing from BinaryWriter configuration.");
}
size_t dim = 1; // default dimension (single item)
if (config.ExistsCurrent(L"dim"))
{
dim = config(L"dim");
}
// get the section type (used for caching)
SectionType sectionType = sectionTypeNull;
if (config.ExistsCurrent(L"sectionType"))
{
SectionType foundType = sectionTypeNull;
wstring type = config(L"sectionType");
for (int i = 0; i < sectionTypeMax; i++)
{
if (EqualCI(type, SectionTypeStrings[i]))
{
foundType = SectionType(i);
break;
}
}
// check to make sure it matched something
if (foundType == sectionTypeNull)
{
InvalidArgument("Invalid value for 'sectionType' in BinaryWriter configuration: %ls", type.c_str());
}
sectionType = foundType;
}
// calculate number of bytes = dim*elementSize*records
size_t dataOnlySize = records * elementSize * dim;
size_t dataSize = dataOnlySize + sectionHeaderMin;
// filename to use the one defined at this level, if there is none use the parent file
SectionFile* file = NULL;
if (config.ExistsCurrent(L"wfile"))
{
std::wstring wfile = config(L"wfile");
auto secFile = m_secFiles.find(wfile);
if (secFile != m_secFiles.end())
{
file = secFile->second;
}
else
{
// TODO: sanity check and use records as a clue of how big to make it
size_t initialSize = config(L"wsize", (size_t) 256); // default to 256MB if not provided
initialSize *= 1024 * 1024; // convert MB to bytes
if (initialSize < dataSize)
initialSize = dataSize * 5 / 4; // make the initalSize slightly larger than needed for data
file = new SectionFile(wfile, fileOptionsReadWrite, initialSize);
m_secFiles[wfile] = file;
parentSection = file->FileSection();
parentSection->SetElementCount(records);
parentSection->SetFileUniqueId(this->m_uniqueID);
}
}
else
{ // no file defined at this config level, use parent file
if (parentSection != NULL && parentSection->GetSectionFile() != NULL)
{
file = parentSection->GetSectionFile();
}
else if (sectionType != sectionTypeNull)
{
InvalidArgument("No filename (wfile) defined in BinaryWriter configuration.");
}
}
// determine file position if needed
size_t filePositionLast = 0;
size_t filePositionNext = 0;
if (file != NULL)
{
// get the next available position in the file (always on the end)
filePositionLast = file->GetFilePositionMax();
filePositionNext = file->RoundUp(filePositionLast);
// we have a gap, zero it out to keep the file clean
if (filePositionLast != filePositionNext)
{
size_t size = filePositionNext - filePositionLast;
size_t roundDown = file->RoundUp(filePositionLast - file->GetViewAlignment() - 1);
// need to get a veiw to zero out non-used bytes
void* view = file->GetView(roundDown, file->GetViewAlignment());
char* ptr = (char*) view + filePositionLast % file->GetViewAlignment();
memset(ptr, 0, size);
file->ReleaseView(view);
}
}
// get the new section name
std::string sectionName = config.ConfigName();
// get the window size, to see if we want to do separate element mapping
size_t windowSize = p_windowSize;
if (config.ExistsCurrent(L"windowSize"))
{
windowSize = config(L"windowSize");
}
MappingType mappingMain = windowSize ? mappingElementWindow : mappingParent;
MappingType mappingAux = windowSize ? mappingSection : mappingParent;
// now create the new section
Section* section = NULL;
switch (sectionType)
{
case sectionTypeNull:
// this happens for the original file header, nothing to do
// also used when multiple files are defined, but none at the base level
break;
case sectionTypeFile: // file header
// shouldn't occur, but same case as above
break;
case sectionTypeData: // data section
section = new Section(file, parentSection, filePositionNext, mappingMain, dataSize);
section->InitHeader(sectionTypeData, sectionName + ":Data Section", sectionDataFloat, sizeof(ElemType));
break;
case sectionTypeLabel: // label data
{
size_t elementSize2 = sizeof(LabelIdType);
dataSize = records * elementSize2 + sectionHeaderMin;
auto sectionLabel = new SectionLabel(file, parentSection, filePositionNext, mappingMain, dataSize);
SectionData dataType = sectionDataInt;
LabelKind labelKind = labelCategory; // default
if (config.Match(L"labelType", L"Regression"))
{
labelKind = labelRegression;
dataType = sectionDataFloat;
elementSize2 = sizeof(ElemType);
}
else if (config.Match(L"labelType", L"Category"))
{
// everything set already, default value
}
else
{
RuntimeError("Invalid type 'labelType' or missing in BinaryWriter configuration.");
}
// initialize the section header
sectionLabel->InitHeader(sectionTypeLabel, sectionName + ":Labels", dataType, (WORD) elementSize2);
// initialize the special label header items
sectionLabel->SetLabelKind(labelKind);
sectionLabel->SetLabelDim(config(L"labelDim"));
section = sectionLabel;
break;
}
case sectionTypeLabelMapping: // label mapping table (array of strings)
section = new SectionString(file, parentSection, filePositionNext, mappingAux, dataSize);
section->InitHeader(sectionTypeLabelMapping, sectionName + ":Label Map", sectionDataStrings, 0); // declare variable length strings
section->SetFlags(flagAuxilarySection);
section->SetFlags(flagVariableSized);
break;
case sectionTypeStats: // data statistics
{
ConfigArray calcStats = config(L"compute");
records = calcStats.size();
elementSize = sizeof(NumericStatistics);
dataOnlySize = records * elementSize;
dataSize = dataOnlySize + sectionHeaderMin;
auto sectionStats = new SectionStats(file, parentSection, filePositionNext, mappingAux, dataSize);
sectionStats->InitHeader(sectionTypeStats, sectionName + ":Data Statistics", sectionDataStruct, sizeof(NumericStatistics)); // declare variable length strings
sectionStats->SetFlags(flagAuxilarySection);
section = sectionStats;
break;
}
case sectionTypeCategoryLabel:
section = new Section(file, parentSection, filePositionNext, mappingMain, dataSize);
section->InitHeader(sectionTypeCategoryLabel, sectionName + ":Category Labels", sectionDataFloat, sizeof(ElemType)); // declare variable length strings
break;
}
// set the rest of the header variables necessary
if (section == NULL)
{
// NULL or file section/already created
section = parentSection;
}
else
{
section->SetElementSize(elementSize);
section->SetElementsPerRecord(dim);
section->SetElementCount(records * dim);
section->SetSize(dataSize);
section->SetSizeAll(dataSize);
// windowSize is in records, convert to bytes
size_t dataWindowSize = windowSize ? windowSize * elementSize * dim : dataOnlySize;
// clamp it down to actual data size
dataWindowSize = min(dataOnlySize, dataWindowSize);
// now get the data pointer setup and allocate the view as necessary
bool auxSection = !!(section->GetFlags() & flagAuxilarySection);
section->EnsureElements(0, auxSection ? dataOnlySize : dataWindowSize);
// update the max file position for the next section
file->SetFilePositionMax(section->GetFilePosition() + dataSize);
// Add new section to parent
parentSection->AddSection(section);
}
// From here on down we have a fully usable section object
// now find the subsections and repeat
vector<std::wstring> subsections;
FindConfigNames(config, "sectionType", subsections);
// look for any children and create them as well
for (std::wstring subsection : subsections)
{
CreateSection(config(subsection), section, records, windowSize);
}
// wait until here so everything is mapped and valid in the object
if (sectionType == sectionTypeStats)
{
ConfigArray calcStats = config(L"compute");
((SectionStats*) section)->InitCompute(calcStats);
}
// add to section map
if (sectionType != sectionTypeFile && sectionType != sectionTypeNull)
{
std::wstring wsectionName = msra::strfun::utf16(sectionName);
// can't have identical names in a write configuration
if (m_sections.find(wsectionName) != m_sections.end())
{
RuntimeError("Identical section name appears twice:%s", sectionName.c_str());
}
m_sections[wsectionName] = section;
}
// validate the header (make sure it's sane)
if (section && file && !section->ValidateHeader(file->Writing()))
{
RuntimeError("Invalid header in file %ls, in header %ls\n", file->GetName().c_str(), section->GetName().c_str());
}
// return the now complete section
return section;
}
void TestCommandLine(const ConfigParameters& configBase)
{
// commandLine=[
ConfigParameters config(configBase("commandline"));
ConfigParameters stringTests(config("stringTests"));
ConfigParameters unicodeTests(stringTests("unicodeTests"));
ConfigParameters arrayTests(config("arrayTests"));
ConfigParameters dictTests(config("dictTests"));
// # config file parsing, basic types
// int=20
int i = config("int", "5555");
// cout << i << endl;
i = config("nothere", "1234");
// cout << i << endl;
// long=8100000
long l = config(L"long", "5555");
// cout << l << endl;
l = config("nothere", "1234");
// size_t=12345678901234
// should get the same thing asking from a double nested config
l = unicodeTests(L"long", "5555");
// cout << l << endl;
l = unicodeTests("nothere", "1234");
// size_t=12345678901234
size_t s = config("size_t", "5555");
// cout << s << endl;
s = config(L"nothere", "1234");
// get stuff from base level config (3 levels down)
string type = unicodeTests("type");
string command = unicodeTests("command");
// boolTrue=t
bool bt = config("boolTrue");
// boolFalse=f
bool bf = config("boolFalse");
// boolImpliedTrue
bool bit = config("boolImpliedTrue");
bit;
bool bif = config.Exists(L"boolImpliedFalse");
bif;
bf = config("nothere", "false");
// cout << bf << endl;
// float=1234.5678
float f = config("float", "555.555");
// cout << f << endl;
f = config("nothere", "1.234");
// double=1.23456e-99
double d = config("double", "555.555");
// cout << d << endl;
d = stringTests("nothere", "1.2345");
// string=string1
std::string str = stringTests("string");
str = stringTests(L"nothere", "default");
// cout << str << endl;
str = stringTests("nothere", "defString");
// cout << str << endl;
// stringQuotes="This is a string with quotes"
str = stringTests("stringQuotes");
// wstring=東京
std::wstring wstr = unicodeTests("wstring");
wstr = (std::wstring) unicodeTests(L"nothere", L"newValue");
// wcout << wstr << endl;
wstr = (std::wstring) unicodeTests("nothere", L"defWstring");
// wstringQuotes="東京に行きましょう． 明日"
std::wstring wstrQuotes = unicodeTests("wstringQuotes");
//
// #array tests
// arrayEmpty={}
ConfigArray arrayEmpty = arrayTests("arrayEmpty");
// arraySingle=hello
ConfigArray arraySingle = arrayTests("arraySingle");
// arrayMultiple=hello;there
ConfigArray arrayMultiple = arrayTests(L"arrayMultiple");
// arrayMultipleBraces={hello:there:with:braces}
ConfigArray arrayMultipleBraces = arrayTests("arrayMultipleBraces");
// arrayMultiple = arrayTests("nothere", arrayMultipleBraces); - no longer supported, can add if we need it
// arrayMultipleSeparatorBraces={|hello|there|with|custom|separator|and|braces}
ConfigArray arrayMultipleSeparatorBraces = arrayTests("arrayMultipleSeparatorBraces");
// arrayQuotedStrings={
// "c:\path with spaces\file.txt"
// "d:\data\Jan 21 1999.my file.txt"
// }
ConfigArray arrayQuotedStrings = arrayTests("arrayQuotedStrings");
str = arrayQuotedStrings[0];
str = arrayQuotedStrings[1];
// arrayRepeat=1*1:2*2:3*3
ConfigArray arrayRepeat = arrayTests("arrayRepeat");
// arrayHetro={string;明日;123;1.234e56;True;dict=first=1;second=2}
ConfigArray arrayHetro = arrayTests("arrayHetro");
str = arrayHetro[0];
wstr = (std::wstring) arrayHetro[1];
i = arrayHetro[2];
d = arrayHetro[3];
bt = arrayHetro[4];
ConfigParameters dict(arrayHetro[5]);
std::string name = dict.Name();
// arrayNested={|1:2:3|first:second:third|t:f:t|{|identical*2|separator*1|nested*3}}
ConfigArray arrayNested = arrayTests(L"arrayNested");
name = arrayNested.Name();
ConfigArray array1 = arrayNested[0];
name = array1.Name();
ConfigArray array2 = arrayNested[1];
name = array2.Name();
ConfigArray array3 = arrayNested[2];
name = array3.Name();
ConfigArray array4 = arrayNested[3];
//
// #dictionary tests
// dictEmpty=[]
ConfigParameters dictEmpty(dictTests("dictEmpty"));
// dictSingle=first=1
ConfigParameters dictSingle(dictTests("dictSingle"));
// dictMultiple=first=hello;second=there
ConfigParameters dictMultiple(dictTests("dictMultiple"));
// dictMultipleBraces=[first=hello;second=there;third=with;forth=braces]
ConfigParameters dictMultipleBraces(dictTests(L"dictMultipleBraces"));
// dictMultiple = dictTests("nothere", dictMultipleBraces); no longer supported, can add if we need
// dictMultipleSeparatorBraces=[|first=hello|second=1|thirdBool|forth=12.345|fifth="quoted string"|sixth=古部|seventh=braces]
ConfigParameters dictMultipleSeparatorBraces(dictTests("dictMultipleSeparatorBraces"));
// dictQuotedStrings=[
// files={
// "c:\path with spaces\file.txt"
// "d:\data\Jan 21 1999.my file.txt"
// }
// mapping="e:\the path\that has\spaces"
// ]
ConfigParameters dictQuotedStrings(dictTests("dictQuotedStrings"));
arrayQuotedStrings = dictQuotedStrings("files");
const char* mapping = dictQuotedStrings("mapping");
mapping;
//
// #super nesting
// dictNested=[
// array={
// dict1=1;dict2=2;dict3=3
// #embedded comment - non-named dictionary
// [
// first=1
// second=2
// third=3
// ]
// {
// [%first=n1%second=n2%third=n3%forth=embedded;old;separator]
// d1=n1;d2=n2;d3=n3
// }
// }
// dict2=[first=1;second=2;third=3]
// dict3=[@
// first=1@second=2
// third=3
// forth=4@fifth=5
// ]
// ]
ConfigParameters dictNested(config("dictNested"));
name = dictNested.Name();
ConfigArray array = dictNested("array");
name = array.Name();
ConfigParameters dictElement1(array[0]);
name = dictElement1.Name();
ConfigParameters dictElement2(array[1]);
name = dictElement2.Name();
ConfigArray arrayNest(array[2]);
name = arrayNest.Name();
ConfigParameters dictNElement1(arrayNest[0]);
name = dictNElement1.Name();
ConfigParameters dictNElement2(arrayNest[1]);
name = dictNElement2.Name();
ConfigParameters dict2(dictNested(L"dict2"));
name = dict2.Name();
ConfigParameters dict3(dictNested("dict3"));
name = dict3.Name();
//]
// File file(L"c:\\temp\\testing.txt", fileOptionsRead | fileOptionsText);
// char marker[5];
// if (file.TryGetMarker(fileMarkerBeginSection, L"BVAL"))
// {
// if (file.TryGetMarker(fileMarkerBeginSection, L"BNEW"))
// {
// if (!file.TryGetMarker(fileMarkerBeginSection, L"OTHER"))
// {
// float val;
// file >> val;
// printf("pass");
// }
// }
// }
// TestConfiguration<ElemType>(configBase);
TestMacros<ElemType>(configBase);
}
