static void DoEvalBase(const ConfigParameters& config, IDataReader& reader) { DEVICEID_TYPE deviceId = DeviceFromConfig(config); ConfigArray minibatchSize = config(L"minibatchSize", "40960"); size_t epochSize = config(L"epochSize", "0"); if (epochSize == 0) { epochSize = requestDataSize; } wstring modelPath = config(L"modelPath"); intargvector mbSize = minibatchSize; int traceLevel = config(L"traceLevel", "0"); size_t numMBsToShowResult = config(L"numMBsToShowResult", "100"); ConfigArray evalNodeNames = config(L"evalNodeNames", ""); vector<wstring> evalNodeNamesVector; for (int i = 0; i < evalNodeNames.size(); ++i) { evalNodeNamesVector.push_back(evalNodeNames[i]); } auto net = ComputationNetwork::CreateFromFile<ElemType>(deviceId, modelPath); SimpleEvaluator<ElemType> eval(net, numMBsToShowResult, traceLevel); eval.Evaluate(&reader, evalNodeNamesVector, mbSize[0], epochSize); }
void ConfigList::load(std::istream &sin, string &line) { for (;;){ ConfigArray *el = create(); push_back(el); el->load(sin, line); if ((line == "[]") || (line == "")) return; } }
int wmain(int argc, wchar_t* argv[]) { try { ConfigParameters config; ConfigParameters::ParseCommandLine(argc, argv, config); // get the command param set they want wstring logpath = config("stderr", L""); ConfigArray command = config("command", "train"); // dump config info fprintf(stderr, "command: "); for (int i = 0; i < command.size(); i++) { fprintf(stderr, "%s ", command[i].c_str()); } // run commands std::string type = config("precision", "float"); // accept old precision key for backward compatibility if (config.Exists("type")) type = config("type", "float"); fprintf(stderr, "\nprecision = %s\n", type.c_str()); if (type == "float") DoCommand<float>(config); else if (type == "double") DoCommand<double>(config); else RuntimeError("invalid precision specified: %s", type.c_str()); } catch (std::exception& err) { fprintf(stderr, "EXCEPTION occurred: %s", err.what()); Microsoft::MSR::CNTK::DebugUtil::PrintCallStack(); #ifdef _DEBUG DebugBreak(); #endif return -1; } catch (...) { fprintf(stderr, "Unknown ERROR occurred"); Microsoft::MSR::CNTK::DebugUtil::PrintCallStack(); #ifdef _DEBUG DebugBreak(); #endif return -1; } return 0; }
static void DisableLegacyUsage(const ConfigParameters& TopLevelConfig, const ConfigArray& commands) { for (size_t i = 0; i < commands.size(); i++) { ConfigParameters cfgParameters(TopLevelConfig(commands[i])); DisableLegacyTruncationSettings(TopLevelConfig, cfgParameters); } }
bool ConfigArray::operator == (const ConfigArray &a) const { list<ConfigValue*>::const_iterator it; for (it = values.begin(); it != values.end(); it++){ const ConfigValue *v = a.getValue((*it)->m_name); if (v == NULL) return false; if (!(*v == *(*it))) return false; } return true; }
ComputationNetworkPtr GetModelFromConfig(const ConfigRecordType& config, const wstring& outputNodeNamesConfig, vector<wstring>& outputNodeNamesVector) { DEVICEID_TYPE deviceId = DeviceFromConfig(config); ConfigArray outputNodeNames = config(outputNodeNamesConfig.c_str(), ConfigArray("")); ComputationNetworkPtr net; // first try if a NetworkBuilder is present function<ComputationNetworkPtr(DEVICEID_TYPE)> createNetworkFn; bool gotIt = TryGetNetworkFactory<ConfigRecordType, ElemType>(config, createNetworkFn); if (gotIt) { // We have several ways to create a network. net = createNetworkFn(deviceId); if (outputNodeNames.size() > 0) { net->InvalidateCompiledNetwork(); PatchOutputNodes(net, outputNodeNames, outputNodeNamesVector); net->CompileNetwork(); // BUGBUG: This will generate double Validation output in the log } } else // no NetworkBuilder given: load from 'modelPath' { wstring modelPath = config(L"modelPath"); // We don't use CreateFromFile() here since the user might specify OutputNodeNames in the config. // By not compiling the network before patching, we avoid double log output for validation. net = make_shared<ComputationNetwork>(deviceId); net->SetTraceLevel(config(L"traceLevel", 0)); net->Read<ElemType>(modelPath); if (outputNodeNames.size() > 0) PatchOutputNodes(net, outputNodeNames, outputNodeNamesVector); net->CompileNetwork(); } return net; }
// special temporary function to guard against a now invalid usage of "truncated" which exists in some IPG production setups static void DisableLegacyTruncationSettings(const ConfigParameters& TopLevelConfig, const ConfigParameters& commandConfig) { if (TopLevelConfig.ExistsCurrent(L"Truncated")) { return; } // if any of the action has set a reader/SGD section and has different Truncated value for reader and SGD section ConfigArray actions = commandConfig(L"action"); for (size_t i = 0; i < actions.size(); i++) { if (actions[i] == "train" || actions[i] == "trainRNN") { ConfigParameters sgd = ConfigParameters(commandConfig(L"SGD")); ConfigParameters reader = ConfigParameters(commandConfig(L"reader")); // reader and SGD sections are two must-have sections in train/trainRNN if (reader.ExistsCurrent(L"Truncated") && !sgd.ExistsCurrent(L"Truncated")) { InvalidArgument("DisableLegacyUsage: setting Truncated only in reader section are not allowed. Please move Truncated=true/false to the top level section."); } } } }
void DoCrossValidate(const ConfigParameters& config) { // test ConfigParameters readerConfig(config(L"reader")); readerConfig.Insert("traceLevel", config(L"traceLevel", "0")); DEVICEID_TYPE deviceId = DeviceFromConfig(config); ConfigArray minibatchSize = config(L"minibatchSize", "40960"); size_t epochSize = config(L"epochSize", "0"); if (epochSize == 0) { epochSize = requestDataSize; } wstring modelPath = config(L"modelPath"); intargvector mbSize = minibatchSize; ConfigArray cvIntervalConfig = config(L"crossValidationInterval"); intargvector cvInterval = cvIntervalConfig; size_t sleepSecondsBetweenRuns = config(L"sleepTimeBetweenRuns", "0"); int traceLevel = config(L"traceLevel", 0); size_t numMBsToShowResult = config(L"numMBsToShowResult", "100"); size_t firstMBsToShowResult = config(L"firstMBsToShowResult", "0"); size_t maxSamplesInRAM = config(L"maxSamplesInRAM", (size_t)SIZE_MAX); size_t numSubminiBatches = config(L"numSubminibatches", (size_t)1); bool enableDistributedMBReading = config(L"distributedMBReading", false); ConfigArray evalNodeNames = config(L"evalNodeNames", ""); vector<wstring> evalNodeNamesVector; for (int i = 0; i < evalNodeNames.size(); ++i) { evalNodeNamesVector.push_back(evalNodeNames[i]); } std::vector<std::vector<EpochCriterion>> cvErrorResults; std::vector<std::wstring> cvModels; DataReader cvDataReader(readerConfig); bool finalModelEvaluated = false; for (size_t i = cvInterval[0]; i <= cvInterval[2]; i += cvInterval[1]) { wstring cvModelPath = msra::strfun::wstrprintf(L"%ls.%lld", modelPath.c_str(), i); if (!fexists(cvModelPath)) { fprintf(stderr, "Model %ls does not exist.\n", cvModelPath.c_str()); if (finalModelEvaluated || !fexists(modelPath)) continue; // file missing else { cvModelPath = modelPath; finalModelEvaluated = true; } } cvModels.push_back(cvModelPath); auto net = ComputationNetwork::CreateFromFile<ElemType>(deviceId, cvModelPath); // BUGBUG: ^^ Should use GetModelFromConfig() SimpleEvaluator<ElemType> eval(net, MPIWrapper::GetInstance(), enableDistributedMBReading, numMBsToShowResult, firstMBsToShowResult, traceLevel, maxSamplesInRAM, numSubminiBatches); fprintf(stderr, "Model %ls --> \n", cvModelPath.c_str()); auto evalErrors = eval.Evaluate(&cvDataReader, evalNodeNamesVector, mbSize[0], epochSize); cvErrorResults.push_back(evalErrors); ::Sleep(1000 * sleepSecondsBetweenRuns); } // find best model if (cvErrorResults.size() == 0) LogicError("No model is evaluated."); vector<double> minErrors; vector<int> minErrIds; vector<EpochCriterion> evalErrors = cvErrorResults[0]; for (int i = 0; i < evalErrors.size(); ++i) { minErrors.push_back(evalErrors[i].Average()); minErrIds.push_back(0); } for (int i = 0; i < cvErrorResults.size(); i++) { evalErrors = cvErrorResults[i]; for (int j = 0; j < evalErrors.size(); j++) { if (evalErrors[j].Average() < minErrors[j]) { minErrors[j] = evalErrors[j].Average(); minErrIds[j] = i; } } } fprintf(stderr, "Best models:\n"); fprintf(stderr, "------------\n"); for (int i = 0; i < minErrors.size(); ++i) fprintf(stderr, "Based on Err[%d]: Best model = %ls with min err %.8g\n", i, cvModels[minErrIds[i]].c_str(), minErrors[i]); }
// called from wmain which is a wrapper that catches & repots Win32 exceptions int wmainOldCNTKConfig(int argc, wchar_t* argv[]) { std::string timestamp = TimeDateStamp(); PrintBanner(argc, argv, timestamp); ConfigParameters config; std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config); // get the command param set they want int traceLevel = config(L"traceLevel", 0); #ifndef CPUONLY ConfigValue val = config("deviceId", "auto"); if (!EqualCI(val, "cpu") && !EqualCI(val, "auto")) { if (static_cast<int>(val) >= 0) // gpu (id >= 0) { CheckSupportForGpu(static_cast<int>(val)); // throws if gpu is not supported } } #endif if (config(L"timestamping", false)) ProgressTracing::SetTimestampingFlag(); if (config(L"forceDeterministicAlgorithms", false)) Globals::ForceDeterministicAlgorithms(); // get the command param set they want wstring logpath = config(L"stderr", L""); wstring doneFile = config(L"doneFile", L""); ConfigArray command = config(L"command", "train"); // parallel training // The top-level 'parallelTrain' is a bool, not to be confused with the parallelTrain block inside SGD. shared_ptr<Microsoft::MSR::CNTK::MPIWrapper> mpi; auto ensureMPIWrapperCleanup = MakeScopeExit(&MPIWrapper::DeleteInstance); // when running under MPI with more than one node, use 'true' as the default value for parallelTrain, // 'false' otherwise. bool paralleltrain = config(L"parallelTrain", (MPIWrapper::GetTotalNumberOfMPINodes() > 1)); if (paralleltrain) { mpi = MPIWrapper::GetInstance(true /*create*/); } g_shareNodeValueMatrices = config(L"shareNodeValueMatrices", false); TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0)); if (logpath != L"") { #if 1 // keep the ability to do it how it was done before 1.8; delete if noone needs it anymore let useOldWay = ProgressTracing::GetTimestampingFlag(); // enable it when running in our server farm if (useOldWay) { for (int i = 0; i < command.size(); i++) // append all 'command' entries { logpath += L"_"; logpath += (wstring)command[i]; } logpath += L".log"; // append .log } if (paralleltrain && useOldWay) { std::wostringstream oss; oss << mpi->CurrentNodeRank(); logpath += L"rank" + oss.str(); } else #endif // for MPI workers except main, append .rankN if (paralleltrain && mpi->CurrentNodeRank() != 0) logpath += msra::strfun::wstrprintf(L".rank%d", mpi->CurrentNodeRank()); RedirectStdErr(logpath); if (traceLevel == 0) PrintBanner(argc, argv, timestamp); // repeat simple banner into log file } // full config info if (traceLevel > 0) { PrintBuiltInfo(); PrintGpuInfo(); } #ifdef _DEBUG if (traceLevel > 0) { // This simply merges all the different config parameters specified (eg, via config files or via command line directly), // and prints it. fprintf(stderr, "\nConfiguration, Raw:\n\n"); LOGPRINTF(stderr, "%s\n", rawConfigString.c_str()); // Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overridden at command line), // All of these assignments will appear, even though only the last assignment matters. fprintf(stderr, "\nConfiguration After Variable Resolution:\n\n"); LOGPRINTF(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str()); } #endif SetMathLibTraceLevel(traceLevel); // This outputs the final value each variable/parameter is assigned to in config (so if a parameter is set multiple times, only the last // value it is set to will appear). if (traceLevel > 0) { fprintf(stderr, "\nConfiguration After Processing and Variable Resolution:\n\n"); config.dumpWithResolvedVariables(); LOGPRINTF(stderr, "Commands:"); for (int i = 0; i < command.size(); i++) fprintf(stderr, " %s", command[i].c_str()); fprintf(stderr, "\n"); } // run commands std::string type = config(L"precision", "float"); // accept old precision key for backward compatibility if (config.Exists("type")) InvalidArgument("CNTK: Use of 'type' parameter is deprecated, it is called 'precision' now."); if (traceLevel > 0) { LOGPRINTF(stderr, "precision = \"%s\"\n", type.c_str()); } if (type == "float") DoCommands<float>(config, mpi); else if (type == "double") DoCommands<double>(config, mpi); else RuntimeError("CNTK: Invalid precision string: \"%s\", must be \"float\" or \"double\"", type.c_str()); // if completed then write a doneFile if requested if (!doneFile.empty()) { FILE* fp = fopenOrDie(doneFile.c_str(), L"w"); fprintf(fp, "Successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str()); fcloseOrDie(fp); } if (ProgressTracing::GetTimestampingFlag()) { LOGPRINTF(stderr, "__COMPLETED__\n"); // running in server environment which expects this string } else fprintf(stderr, "COMPLETED.\n"); fflush(stderr); return EXIT_SUCCESS; }
void DoCommands(const ConfigParameters& config, const shared_ptr<MPIWrapper>& mpi) { ConfigArray command = config(L"command", "train"); if (Globals::ShouldForceDeterministicAlgorithms()) ForceDeterministicAlgorithmsOnCPU(); else { // Setting specified number of threads. int numCPUThreads = config(L"numCPUThreads", "0"); numCPUThreads = CPUMatrix<ElemType>::SetNumThreads(numCPUThreads); if (numCPUThreads > 0) { LOGPRINTF(stderr, "Using %d CPU threads.\n", numCPUThreads); } } bool progressTracing = config(L"progressTracing", false); // temporary hack to prevent users from failing due to a small breaking change related to the "truncated" flag (will be redone bigger and better some day) DisableLegacyUsage(config, command); // summarize command info upfront in the log and stdout size_t fullTotalMaxEpochs = 0; for (int i = 0; i < command.size(); i++) { // get the configuration parameters that match the command ConfigParameters commandParams(config(command[i])); ConfigArray action = commandParams("action", "train"); // determine the action to perform, and do it for (int j = 0; j < action.size(); j++) { if (action[j] == "train" || action[j] == "trainRNN") { wstring modelPath = commandParams("modelPath"); size_t maxEpochs = GetMaxEpochs(commandParams); if (progressTracing) { LOGPRINTF(stderr, "CNTKModelPath: %ls\n", modelPath.c_str()); LOGPRINTF(stderr, "CNTKCommandTrainInfo: %s : %d\n", command[i].c_str(), (int)maxEpochs); } fullTotalMaxEpochs += maxEpochs; } } } if (progressTracing) { LOGPRINTF(stderr, "CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : %d\n", (int)fullTotalMaxEpochs); } // set up progress tracing for compute cluster management if (progressTracing && (!mpi || mpi->IsMainNode())) { ProgressTracing::SetTracingFlag(); ProgressTracing::TraceTotalNumberOfSteps(fullTotalMaxEpochs); // enable tracing, using this as the total number of epochs } size_t fullEpochsOffset = 0; // execute the commands for (int i = 0; i < command.size(); i++) { // get the configuration parameters that match the command const string thisCommand = command[i]; ConfigParameters commandParams(config(thisCommand)); ConfigArray action = commandParams("action", "train"); int traceLevel = commandParams("traceLevel", "0"); if (progressTracing && ((mpi == nullptr) || mpi->IsMainNode())) { ProgressTracing::SetStepOffset(fullEpochsOffset); // this is the epoch number that SGD will log relative to } // determine the action to perform, and do it for (int j = 0; j < action.size(); j++) { const string thisAction = action[j]; // print a banner to visually separate each action in the log const char* delim = "##############################################################################"; string showActionAs = thisCommand + " command (" + thisAction + " action)"; fprintf(stderr, "\n"); LOGPRINTF(stderr, "%s\n", delim); LOGPRINTF(stderr, "#%*s#\n", (int)(strlen(delim) - 2), ""); LOGPRINTF(stderr, "# %s%*s #\n", showActionAs.c_str(), (int)(strlen(delim) - showActionAs.size() - 4), ""); LOGPRINTF(stderr, "#%*s#\n", (int)(strlen(delim) - 2), ""); LOGPRINTF(stderr, "%s\n\n", delim); if ((mpi == nullptr) || (commandstoRunOnAllRanks.find(thisAction) != commandstoRunOnAllRanks.end()) || mpi->IsMainNode()) { if (thisAction == "train" || thisAction == "trainRNN") { if (progressTracing) { LOGPRINTF(stderr, "CNTKCommandTrainBegin: %s\n", command[i].c_str()); } DoTrain<ConfigParameters, ElemType>(commandParams); if (progressTracing) { LOGPRINTF(stderr, "CNTKCommandTrainEnd: %s\n", command[i].c_str()); } fullEpochsOffset += GetMaxEpochs(commandParams); } // TODO: Choose a clearer name. else if (thisAction == "pbn") { DoEvalBN<ElemType>(commandParams); } else if (thisAction == "adapt") { DoAdapt<ElemType>(commandParams); } else if (thisAction == "test" || thisAction == "eval") { DoEval<ElemType>(commandParams); } else if (thisAction == "edit") { DoEdit<ElemType>(commandParams); } else if (thisAction == "cv") { DoCrossValidate<ElemType>(commandParams); } else if (thisAction == "write") { DoWriteOutput<ElemType>(commandParams); } else if (thisAction == "devtest") { TestCn<ElemType>(config); // for "devtest" action pass the root config instead } else if (thisAction == "dumpNodes" /*deprecated:*/ || thisAction == "dumpNode" || thisAction == "dumpnode") { DoDumpNodes<ElemType>(commandParams); } else if (thisAction == "convertdbn") { DoConvertFromDbn<ElemType>(commandParams); } else if (thisAction == "exportdbn") { DoExportToDbn<ElemType>(commandParams); } else if (thisAction == "createLabelMap") { DoCreateLabelMap<ElemType>(commandParams); } else if (thisAction == "writeWordAndClass") { DoWriteWordAndClassInfo<ElemType>(commandParams); } else if (thisAction == "plot") { DoTopologyPlot<ElemType>(commandParams); } else if (thisAction == "SVD") { DoParameterSVD<ElemType>(commandParams); } else { RuntimeError("unknown action: %s in command set: %s", thisAction.c_str(), command[i].c_str()); } } fprintf(stderr, "\n"); if (traceLevel > 0) { LOGPRINTF(stderr, "Action \"%s\" complete.\n\n", thisAction.c_str()); } NDLScript<ElemType> ndlScript; ndlScript.ClearGlobal(); // clear global macros between commands // Synchronize all ranks before proceeding to next action/command if (mpi) mpi->WaitAll(); } } }
void DoWriteOutput(const ConfigParameters& config) { ConfigParameters readerConfig(config(L"reader")); readerConfig.Insert("traceLevel", config(L"traceLevel", "0")); readerConfig.Insert("randomize", "None"); // we don't want randomization when output results DataReader testDataReader(readerConfig); DEVICEID_TYPE deviceId = DeviceFromConfig(config); ConfigArray minibatchSize = config(L"minibatchSize", "2048"); wstring modelPath = config(L"modelPath"); intargvector mbSize = minibatchSize; size_t epochSize = config(L"epochSize", "0"); if (epochSize == 0) { epochSize = requestDataSize; } ConfigArray outputNodeNames = config(L"outputNodeNames", ""); vector<wstring> outputNodeNamesVector; // Note this is required since the user might specify OutputNodeNames in the config, so don't use CreateFromFile, // instead we build the network ourselves. auto net = make_shared<ComputationNetwork>(deviceId); net->Read<ElemType>(modelPath); if (outputNodeNames.size() > 0) { net->OutputNodes().clear(); for (int i = 0; i < outputNodeNames.size(); ++i) { outputNodeNamesVector.push_back(outputNodeNames[i]); net->OutputNodes().emplace_back(net->GetNodeFromName(outputNodeNames[i])); } } net->CompileNetwork(); SimpleOutputWriter<ElemType> writer(net, 1); if (config.Exists("writer")) { ConfigParameters writerConfig(config(L"writer")); bool bWriterUnittest = writerConfig(L"unittest", "false"); DataWriter testDataWriter(writerConfig); writer.WriteOutput(testDataReader, mbSize[0], testDataWriter, outputNodeNamesVector, epochSize, bWriterUnittest); } else if (config.Exists("outputPath")) { wstring outputPath = config(L"outputPath"); // gather additional formatting options typename decltype(writer)::WriteFormattingOptions formattingOptions; if (config.Exists("format")) { ConfigParameters formatConfig(config(L"format")); if (formatConfig.ExistsCurrent("type")) // do not inherit 'type' from outer block { string type = formatConfig(L"type"); if (type == "real") formattingOptions.isCategoryLabel = false; else if (type == "category") formattingOptions.isCategoryLabel = true; else InvalidArgument("write: type must be 'real' or 'category'"); if (formattingOptions.isCategoryLabel) formattingOptions.labelMappingFile = (wstring)formatConfig(L"labelMappingFile", L""); } formattingOptions.transpose = formatConfig(L"transpose", formattingOptions.transpose); formattingOptions.prologue = formatConfig(L"prologue", formattingOptions.prologue); formattingOptions.epilogue = formatConfig(L"epilogue", formattingOptions.epilogue); formattingOptions.sequenceSeparator = formatConfig(L"sequenceSeparator", formattingOptions.sequenceSeparator); formattingOptions.sequencePrologue = formatConfig(L"sequencePrologue", formattingOptions.sequencePrologue); formattingOptions.sequenceEpilogue = formatConfig(L"sequenceEpilogue", formattingOptions.sequenceEpilogue); formattingOptions.elementSeparator = formatConfig(L"elementSeparator", formattingOptions.elementSeparator); formattingOptions.sampleSeparator = formatConfig(L"sampleSeparator", formattingOptions.sampleSeparator); formattingOptions.precisionFormat = formatConfig(L"precisionFormat", formattingOptions.precisionFormat); } writer.WriteOutput(testDataReader, mbSize[0], outputPath, outputNodeNamesVector, formattingOptions, epochSize); } else InvalidArgument("write command: You must specify either 'writer'or 'outputPath'"); }
int wmainOldCNTKConfig(int argc, wchar_t* argv[]) // called from wmain which is a wrapper that catches & repots Win32 exceptions { ConfigParameters config; std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config); // get the command param set they want wstring logpath = config(L"stderr", L""); // [1/26/2015 erw, add done file so that it can be used on HPC] wstring DoneFile = config(L"DoneFile", L""); ConfigArray command = config(L"command", "train"); // paralleltrain training g_mpi = nullptr; bool paralleltrain = config(L"parallelTrain", "false"); if (paralleltrain) { g_mpi = new MPIWrapper(); } g_shareNodeValueMatrices = config(L"shareNodeValueMatrices", false); TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0)); if (logpath != L"") { for (int i = 0; i < command.size(); i++) { logpath += L"_"; logpath += (wstring) command[i]; } logpath += L".log"; if (paralleltrain) { std::wostringstream oss; oss << g_mpi->CurrentNodeRank(); logpath += L"rank" + oss.str(); } RedirectStdErr(logpath); } PrintBuiltInfo(); // this one goes to log file std::string timestamp = TimeDateStamp(); // dump config info fprintf(stderr, "running on %s at %s\n", GetHostName().c_str(), timestamp.c_str()); fprintf(stderr, "command line: \n"); for (int i = 0; i < argc; i++) { fprintf(stderr, "%s ", WCharToString(argv[i]).c_str()); } // This simply merges all the different config parameters specified (eg, via config files or via command line directly), // and prints it. fprintf(stderr, "\n\n>>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>>\n"); fprintf(stderr, "%s\n", rawConfigString.c_str()); fprintf(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<<\n"); // Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overriden at command line), // All of these assignments will appear, even though only the last assignment matters. fprintf(stderr, "\n>>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n"); fprintf(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str()); fprintf(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n"); // This outputs the final value each variable/parameter is assigned to in config (so if a parameter is set multiple times, only the last // value it is set to will appear). fprintf(stderr, "\n>>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n"); config.dumpWithResolvedVariables(); fprintf(stderr, "<<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n"); fprintf(stderr, "command: "); for (int i = 0; i < command.size(); i++) { fprintf(stderr, "%s ", command[i].c_str()); } // run commands std::string type = config(L"precision", "float"); // accept old precision key for backward compatibility if (config.Exists("type")) { type = config(L"type", "float"); } fprintf(stderr, "\nprecision = %s\n", type.c_str()); if (type == "float") { DoCommands<float>(config); } else if (type == "double") { DoCommands<double>(config); } else { RuntimeError("invalid precision specified: %s", type.c_str()); } // still here , write a DoneFile if necessary if (!DoneFile.empty()) { FILE* fp = fopenOrDie(DoneFile.c_str(), L"w"); fprintf(fp, "successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str()); fcloseOrDie(fp); } fprintf(stderr, "COMPLETED\n"), fflush(stderr); delete g_mpi; return EXIT_SUCCESS; }
void DoCommands(const ConfigParameters& config) { ConfigArray command = config(L"command", "train"); int numCPUThreads = config(L"numCPUThreads", "0"); numCPUThreads = CPUMatrix<ElemType>::SetNumThreads(numCPUThreads); if (numCPUThreads > 0) { std::cerr << "Using " << numCPUThreads << " CPU threads" << endl; } bool progressTracing = config(L"progressTracing", false); // temporary hack to prevent users from failling for a small breaking change related to the "truncated" flag (will be redone bigger and better some day) DisableLegacyUsage(config, command); // summarize command info upfront in the log and stdout size_t fullTotalMaxEpochs = 0; for (int i = 0; i < command.size(); i++) { // get the configuration parameters that match the command ConfigParameters commandParams(config(command[i])); ConfigArray action = commandParams("action", "train"); // determine the action to perform, and do it for (int j = 0; j < action.size(); j++) { if (action[j] == "train" || action[j] == "trainRNN") { wstring modelPath = commandParams("modelPath"); std::wcerr << "CNTKModelPath: " << modelPath << endl; size_t maxEpochs = GetMaxEpochs(commandParams); std::cerr << "CNTKCommandTrainInfo: " + command[i] << " : " << maxEpochs << endl; fullTotalMaxEpochs += maxEpochs; } } } std::cerr << "CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : " << fullTotalMaxEpochs << endl; // set up progress tracing for compute cluster management if (progressTracing && ((g_mpi == nullptr) || g_mpi->IsMainNode())) { ProgressTracing::TraceTotalNumberOfSteps(fullTotalMaxEpochs); // enable tracing, using this as the total number of epochs } size_t fullEpochsOffset = 0; // execute the commands for (int i = 0; i < command.size(); i++) { // get the configuration parameters that match the command ConfigParameters commandParams(config(command[i])); ConfigArray action = commandParams("action", "train"); if (progressTracing && ((g_mpi == nullptr) || g_mpi->IsMainNode())) { ProgressTracing::SetStepOffset(fullEpochsOffset); // this is the epoch number that SGD will log relative to } // determine the action to perform, and do it for (int j = 0; j < action.size(); j++) { if (action[j] == "train" || action[j] == "trainRNN") { std::cerr << "CNTKCommandTrainBegin: " + command[i] << endl; DoTrain<ConfigParameters, ElemType>(commandParams); std::cerr << "CNTKCommandTrainEnd: " + command[i] << endl; fullEpochsOffset += GetMaxEpochs(commandParams); } else if (action[j] == "adapt") { DoAdapt<ElemType>(commandParams); } else if (action[j] == "test" || action[j] == "eval") { DoEval<ElemType>(commandParams); } else if (action[j] == "edit") { DoEdit<ElemType>(commandParams); } else if (action[j] == "cv") { DoCrossValidate<ElemType>(commandParams); } else if (action[j] == "write") { DoWriteOutput<ElemType>(commandParams); } else if (action[j] == "devtest") { TestCn<ElemType>(config); // for "devtest" action pass the root config instead } else if (action[j] == "dumpnode") { DumpNodeInfo<ElemType>(commandParams); } else if (action[j] == "convertdbn") { DoConvertFromDbn<ElemType>(commandParams); } else if (action[j] == "createLabelMap") { DoCreateLabelMap<ElemType>(commandParams); } else if (action[j] == "writeWordAndClass") { DoWriteWordAndClassInfo<ElemType>(commandParams); } else if (action[j] == "plot") { DoTopologyPlot<ElemType>(commandParams); } else if (action[j] == "SVD") { DoParameterSVD<ElemType>(commandParams); } else { RuntimeError("unknown action: %s in command set: %s", action[j].c_str(), command[i].c_str()); } NDLScript<ElemType> ndlScript; ndlScript.ClearGlobal(); // clear global macros between commands } } }
// --------------------------------------------------------------------------- // main() for old CNTK config language // --------------------------------------------------------------------------- // called from wmain which is a wrapper that catches & repots Win32 exceptions int wmainOldCNTKConfig(int argc, wchar_t* argv[]) { ConfigParameters config; std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config); // get the command param set they want bool timestamping = config(L"timestamping", false); if (timestamping) { ProgressTracing::SetTimestampingFlag(); } // get the command param set they want wstring logpath = config(L"stderr", L""); // [1/26/2015 erw, add done file so that it can be used on HPC] wstring DoneFile = config(L"DoneFile", L""); ConfigArray command = config(L"command", "train"); // paralleltrain training shared_ptr<Microsoft::MSR::CNTK::MPIWrapper> mpi; bool paralleltrain = config(L"parallelTrain", "false"); if (paralleltrain) mpi = MPIWrapper::GetInstance(true /*create*/); g_shareNodeValueMatrices = config(L"shareNodeValueMatrices", false); TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0)); if (logpath != L"") { for (int i = 0; i < command.size(); i++) { logpath += L"_"; logpath += (wstring) command[i]; } logpath += L".log"; if (paralleltrain) { std::wostringstream oss; oss << mpi->CurrentNodeRank(); logpath += L"rank" + oss.str(); } RedirectStdErr(logpath); } PrintBuiltInfo(); // this one goes to log file std::string timestamp = TimeDateStamp(); // dump config info fprintf(stderr, "\n"); LOGPRINTF(stderr, "Running on %s at %s\n", GetHostName().c_str(), timestamp.c_str()); LOGPRINTF(stderr, "Command line: \n"); for (int i = 0; i < argc; i++) fprintf(stderr, "%*s%ls", i > 0 ? 2 : 0, "", argv[i]); // use 2 spaces for better visual separability fprintf(stderr, "\n\n"); #if 1 //def _DEBUG // This simply merges all the different config parameters specified (eg, via config files or via command line directly), // and prints it. fprintf(stderr, "\n\n"); LOGPRINTF(stderr, ">>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>>\n"); LOGPRINTF(stderr, "%s\n", rawConfigString.c_str()); LOGPRINTF(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<<\n"); // Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overridden at command line), // All of these assignments will appear, even though only the last assignment matters. fprintf(stderr, "\n"); LOGPRINTF(stderr, ">>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n"); LOGPRINTF(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str()); LOGPRINTF(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n"); // This outputs the final value each variable/parameter is assigned to in config (so if a parameter is set multiple times, only the last // value it is set to will appear). fprintf(stderr, "\n"); LOGPRINTF(stderr, ">>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n"); config.dumpWithResolvedVariables(); LOGPRINTF(stderr, "<<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n"); #endif LOGPRINTF(stderr, "Commands:"); for (int i = 0; i < command.size(); i++) fprintf(stderr, " %s", command[i].c_str()); fprintf(stderr, "\n"); // run commands std::string type = config(L"precision", "float"); // accept old precision key for backward compatibility if (config.Exists("type")) InvalidArgument("CNTK: Use of 'type' parameter is deprecated, it is called 'precision' now."); LOGPRINTF(stderr, "Precision = \"%s\"\n", type.c_str()); if (type == "float") DoCommands<float>(config, mpi); else if (type == "double") DoCommands<double>(config, mpi); else RuntimeError("CNTK: Invalid precision string: \"%s\", must be \"float\" or \"double\"", type.c_str()); // if completed then write a DoneFile if requested if (!DoneFile.empty()) { FILE* fp = fopenOrDie(DoneFile.c_str(), L"w"); fprintf(fp, "successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str()); fcloseOrDie(fp); } // TODO: Change back to COMPLETED (no underscores) LOGPRINTF(stderr, "__COMPLETED__\n"); fflush(stderr); MPIWrapper::DeleteInstance(); return EXIT_SUCCESS; }
void TestConfiguration(const ConfigParameters& configBase) { ConfigParameters configMacros = configBase("macroExample"); for (auto iterMacro = configMacros.begin(); iterMacro != configMacros.end(); iterMacro++) { std::map<std::string, ConfigValue> paramsMap; ConfigParameters configCN = iterMacro->second; if (configCN.Exists("parameters")) { ConfigArray params = configCN("parameters"); for (int i = 0; i < params.size(); ++i) paramsMap[params[i]] = ConfigValue("uninitialized"); } ConfigParameters configNodes = configCN("NodeList"); for (auto iter = configNodes.begin(); iter != configNodes.end(); iter++) { std::wstring nodeName; nodeName = msra::strfun::utf16(iter->first); ConfigArray configNode = iter->second; std::string opName = configNode[0]; if (IsParameter(paramsMap, opName)) { ; } if (opName == "InputValue" && configNode.size() >= 2) { size_t rows = 0; if (!IsParameter(paramsMap, configNode[1])) rows = configNode[1]; } else if (opName == "LearnableParameter" && configNode.size() >= 3) { size_t rows = 0; if (!IsParameter(paramsMap, configNode[1])) rows = configNode[1]; size_t cols = 0; if (!IsParameter(paramsMap, configNode[2])) cols = configNode[2]; bool learningRateMultiplier = 0; bool init = false; ConfigArray initData; // look for optional parameters for (int i = 3; i < configNode.size(); ++i) { bool needsGradient = false; ConfigParameters configParam = configNode[i]; if (configParam.Exists("learningRateMultiplier")) // TODO: should this be a test for 'true' rather than Exists()? needsGradient = (float)configParam("learningRateMultiplier") > 0? true : false; else if (configParam.Exists("init")) { init = true; initData = configParam["init"]; } } // if initializing, do so now if (init) { bool uniform = true; ElemType initValueScale = 1; size_t inputSize = cols; if (initData.size() > 0) initValueScale = initData[0]; if (initData.size() > 1) uniform = EqualCI(initData[1], "uniform"); } } } // now link up all the nodes configNodes = configCN("Relation"); for (auto iter = configNodes.begin(); iter != configNodes.end(); iter++) { std::wstring nodeName = msra::strfun::utf16(iter->first); ConfigArray configNode = iter->second; int numChildren = (int) configNode.size(); for (int i = 0; i < numChildren; ++i) { std::wstring nodeName = configNode[i]; } } ConfigParameters configRoots = configCN("RootNodes"); ConfigArray configNode = configRoots("FeatureNodes"); for (size_t i = 0; i < configNode.size(); i++) { std::wstring nodeName = configNode[i]; } if (configRoots.Exists("LabelNodes")) { configNode = configRoots("LabelNodes"); for (size_t i = 0; i < configNode.size(); i++) { std::wstring nodeName = configNode[i]; } } if (configRoots.Exists("CriterionNodes")) { configNode = configRoots("CriterionNodes"); for (size_t i = 0; i < configNode.size(); i++) { std::wstring nodeName = configNode[i]; } } if (configRoots.Exists("CriteriaNodes")) // legacy { configNode = configRoots("CriteriaNodes"); for (size_t i = 0; i < configNode.size(); i++) { std::wstring nodeName = configNode[i]; } } if (configRoots.Exists("NodesReqMultiSeqHandling")) { configNode = configRoots("NodesReqMultiSeqHandling"); for (size_t i = 0; i < configNode.size(); i++) { std::wstring nodeName = configNode[i]; } fprintf(stderr, "WARNING: 'NodesReqMultiSeqHandling' flag is defunct\n"); } if (configRoots.Exists("EvalNodes")) { configNode = configRoots("EvalNodes"); for (size_t i = 0; i < configNode.size(); i++) { std::wstring nodeName = configNode[i]; } } if (configRoots.Exists("OutputNodes")) { configNode = configRoots("OutputNodes"); for (size_t i = 0; i < configNode.size(); i++) { std::wstring nodeName = configNode[i]; } } } }
Section* BinaryWriter<ElemType>::CreateSection(const ConfigParameters& config, Section* parentSection, size_t p_records, size_t p_windowSize) { // first check if we need to open a new section file std::vector<std::wstring> sections; // determine the element size, default to ElemType size size_t elementSize = sizeof(ElemType); if (config.ExistsCurrent(L"elementSize")) { elementSize = config(L"elementSize"); } // get the number of records we should expect (max) // if defined in previous levels same number will be used size_t records = p_records; if (config.ExistsCurrent(L"wrecords")) { records = config(L"wrecords"); } if (records == 0) { InvalidArgument("Required config variable 'wrecords' missing from BinaryWriter configuration."); } size_t dim = 1; // default dimension (single item) if (config.ExistsCurrent(L"dim")) { dim = config(L"dim"); } // get the section type (used for caching) SectionType sectionType = sectionTypeNull; if (config.ExistsCurrent(L"sectionType")) { SectionType foundType = sectionTypeNull; wstring type = config(L"sectionType"); for (int i = 0; i < sectionTypeMax; i++) { if (EqualCI(type, SectionTypeStrings[i])) { foundType = SectionType(i); break; } } // check to make sure it matched something if (foundType == sectionTypeNull) { InvalidArgument("Invalid value for 'sectionType' in BinaryWriter configuration: %ls", type.c_str()); } sectionType = foundType; } // calculate number of bytes = dim*elementSize*records size_t dataOnlySize = records * elementSize * dim; size_t dataSize = dataOnlySize + sectionHeaderMin; // filename to use the one defined at this level, if there is none use the parent file SectionFile* file = NULL; if (config.ExistsCurrent(L"wfile")) { std::wstring wfile = config(L"wfile"); auto secFile = m_secFiles.find(wfile); if (secFile != m_secFiles.end()) { file = secFile->second; } else { // TODO: sanity check and use records as a clue of how big to make it size_t initialSize = config(L"wsize", (size_t) 256); // default to 256MB if not provided initialSize *= 1024 * 1024; // convert MB to bytes if (initialSize < dataSize) initialSize = dataSize * 5 / 4; // make the initalSize slightly larger than needed for data file = new SectionFile(wfile, fileOptionsReadWrite, initialSize); m_secFiles[wfile] = file; parentSection = file->FileSection(); parentSection->SetElementCount(records); parentSection->SetFileUniqueId(this->m_uniqueID); } } else { // no file defined at this config level, use parent file if (parentSection != NULL && parentSection->GetSectionFile() != NULL) { file = parentSection->GetSectionFile(); } else if (sectionType != sectionTypeNull) { InvalidArgument("No filename (wfile) defined in BinaryWriter configuration."); } } // determine file position if needed size_t filePositionLast = 0; size_t filePositionNext = 0; if (file != NULL) { // get the next available position in the file (always on the end) filePositionLast = file->GetFilePositionMax(); filePositionNext = file->RoundUp(filePositionLast); // we have a gap, zero it out to keep the file clean if (filePositionLast != filePositionNext) { size_t size = filePositionNext - filePositionLast; size_t roundDown = file->RoundUp(filePositionLast - file->GetViewAlignment() - 1); // need to get a veiw to zero out non-used bytes void* view = file->GetView(roundDown, file->GetViewAlignment()); char* ptr = (char*) view + filePositionLast % file->GetViewAlignment(); memset(ptr, 0, size); file->ReleaseView(view); } } // get the new section name std::string sectionName = config.ConfigName(); // get the window size, to see if we want to do separate element mapping size_t windowSize = p_windowSize; if (config.ExistsCurrent(L"windowSize")) { windowSize = config(L"windowSize"); } MappingType mappingMain = windowSize ? mappingElementWindow : mappingParent; MappingType mappingAux = windowSize ? mappingSection : mappingParent; // now create the new section Section* section = NULL; switch (sectionType) { case sectionTypeNull: // this happens for the original file header, nothing to do // also used when multiple files are defined, but none at the base level break; case sectionTypeFile: // file header // shouldn't occur, but same case as above break; case sectionTypeData: // data section section = new Section(file, parentSection, filePositionNext, mappingMain, dataSize); section->InitHeader(sectionTypeData, sectionName + ":Data Section", sectionDataFloat, sizeof(ElemType)); break; case sectionTypeLabel: // label data { size_t elementSize2 = sizeof(LabelIdType); dataSize = records * elementSize2 + sectionHeaderMin; auto sectionLabel = new SectionLabel(file, parentSection, filePositionNext, mappingMain, dataSize); SectionData dataType = sectionDataInt; LabelKind labelKind = labelCategory; // default if (config.Match(L"labelType", L"Regression")) { labelKind = labelRegression; dataType = sectionDataFloat; elementSize2 = sizeof(ElemType); } else if (config.Match(L"labelType", L"Category")) { // everything set already, default value } else { RuntimeError("Invalid type 'labelType' or missing in BinaryWriter configuration."); } // initialize the section header sectionLabel->InitHeader(sectionTypeLabel, sectionName + ":Labels", dataType, (WORD) elementSize2); // initialize the special label header items sectionLabel->SetLabelKind(labelKind); sectionLabel->SetLabelDim(config(L"labelDim")); section = sectionLabel; break; } case sectionTypeLabelMapping: // label mapping table (array of strings) section = new SectionString(file, parentSection, filePositionNext, mappingAux, dataSize); section->InitHeader(sectionTypeLabelMapping, sectionName + ":Label Map", sectionDataStrings, 0); // declare variable length strings section->SetFlags(flagAuxilarySection); section->SetFlags(flagVariableSized); break; case sectionTypeStats: // data statistics { ConfigArray calcStats = config(L"compute"); records = calcStats.size(); elementSize = sizeof(NumericStatistics); dataOnlySize = records * elementSize; dataSize = dataOnlySize + sectionHeaderMin; auto sectionStats = new SectionStats(file, parentSection, filePositionNext, mappingAux, dataSize); sectionStats->InitHeader(sectionTypeStats, sectionName + ":Data Statistics", sectionDataStruct, sizeof(NumericStatistics)); // declare variable length strings sectionStats->SetFlags(flagAuxilarySection); section = sectionStats; break; } case sectionTypeCategoryLabel: section = new Section(file, parentSection, filePositionNext, mappingMain, dataSize); section->InitHeader(sectionTypeCategoryLabel, sectionName + ":Category Labels", sectionDataFloat, sizeof(ElemType)); // declare variable length strings break; } // set the rest of the header variables necessary if (section == NULL) { // NULL or file section/already created section = parentSection; } else { section->SetElementSize(elementSize); section->SetElementsPerRecord(dim); section->SetElementCount(records * dim); section->SetSize(dataSize); section->SetSizeAll(dataSize); // windowSize is in records, convert to bytes size_t dataWindowSize = windowSize ? windowSize * elementSize * dim : dataOnlySize; // clamp it down to actual data size dataWindowSize = min(dataOnlySize, dataWindowSize); // now get the data pointer setup and allocate the view as necessary bool auxSection = !!(section->GetFlags() & flagAuxilarySection); section->EnsureElements(0, auxSection ? dataOnlySize : dataWindowSize); // update the max file position for the next section file->SetFilePositionMax(section->GetFilePosition() + dataSize); // Add new section to parent parentSection->AddSection(section); } // From here on down we have a fully usable section object // now find the subsections and repeat vector<std::wstring> subsections; FindConfigNames(config, "sectionType", subsections); // look for any children and create them as well for (std::wstring subsection : subsections) { CreateSection(config(subsection), section, records, windowSize); } // wait until here so everything is mapped and valid in the object if (sectionType == sectionTypeStats) { ConfigArray calcStats = config(L"compute"); ((SectionStats*) section)->InitCompute(calcStats); } // add to section map if (sectionType != sectionTypeFile && sectionType != sectionTypeNull) { std::wstring wsectionName = msra::strfun::utf16(sectionName); // can't have identical names in a write configuration if (m_sections.find(wsectionName) != m_sections.end()) { RuntimeError("Identical section name appears twice:%s", sectionName.c_str()); } m_sections[wsectionName] = section; } // validate the header (make sure it's sane) if (section && file && !section->ValidateHeader(file->Writing())) { RuntimeError("Invalid header in file %ls, in header %ls\n", file->GetName().c_str(), section->GetName().c_str()); } // return the now complete section return section; }
void TestCommandLine(const ConfigParameters& configBase) { // commandLine=[ ConfigParameters config(configBase("commandline")); ConfigParameters stringTests(config("stringTests")); ConfigParameters unicodeTests(stringTests("unicodeTests")); ConfigParameters arrayTests(config("arrayTests")); ConfigParameters dictTests(config("dictTests")); // # config file parsing, basic types // int=20 int i = config("int", "5555"); // cout << i << endl; i = config("nothere", "1234"); // cout << i << endl; // long=8100000 long l = config(L"long", "5555"); // cout << l << endl; l = config("nothere", "1234"); // size_t=12345678901234 // should get the same thing asking from a double nested config l = unicodeTests(L"long", "5555"); // cout << l << endl; l = unicodeTests("nothere", "1234"); // size_t=12345678901234 size_t s = config("size_t", "5555"); // cout << s << endl; s = config(L"nothere", "1234"); // get stuff from base level config (3 levels down) string type = unicodeTests("type"); string command = unicodeTests("command"); // boolTrue=t bool bt = config("boolTrue"); // boolFalse=f bool bf = config("boolFalse"); // boolImpliedTrue bool bit = config("boolImpliedTrue"); bit; bool bif = config.Exists(L"boolImpliedFalse"); bif; bf = config("nothere", "false"); // cout << bf << endl; // float=1234.5678 float f = config("float", "555.555"); // cout << f << endl; f = config("nothere", "1.234"); // double=1.23456e-99 double d = config("double", "555.555"); // cout << d << endl; d = stringTests("nothere", "1.2345"); // string=string1 std::string str = stringTests("string"); str = stringTests(L"nothere", "default"); // cout << str << endl; str = stringTests("nothere", "defString"); // cout << str << endl; // stringQuotes="This is a string with quotes" str = stringTests("stringQuotes"); // wstring=東京 std::wstring wstr = unicodeTests("wstring"); wstr = (std::wstring) unicodeTests(L"nothere", L"newValue"); // wcout << wstr << endl; wstr = (std::wstring) unicodeTests("nothere", L"defWstring"); // wstringQuotes="東京に行きましょう. 明日" std::wstring wstrQuotes = unicodeTests("wstringQuotes"); // // #array tests // arrayEmpty={} ConfigArray arrayEmpty = arrayTests("arrayEmpty"); // arraySingle=hello ConfigArray arraySingle = arrayTests("arraySingle"); // arrayMultiple=hello;there ConfigArray arrayMultiple = arrayTests(L"arrayMultiple"); // arrayMultipleBraces={hello:there:with:braces} ConfigArray arrayMultipleBraces = arrayTests("arrayMultipleBraces"); // arrayMultiple = arrayTests("nothere", arrayMultipleBraces); - no longer supported, can add if we need it // arrayMultipleSeparatorBraces={|hello|there|with|custom|separator|and|braces} ConfigArray arrayMultipleSeparatorBraces = arrayTests("arrayMultipleSeparatorBraces"); // arrayQuotedStrings={ // "c:\path with spaces\file.txt" // "d:\data\Jan 21 1999.my file.txt" // } ConfigArray arrayQuotedStrings = arrayTests("arrayQuotedStrings"); str = arrayQuotedStrings[0]; str = arrayQuotedStrings[1]; // arrayRepeat=1*1:2*2:3*3 ConfigArray arrayRepeat = arrayTests("arrayRepeat"); // arrayHetro={string;明日;123;1.234e56;True;dict=first=1;second=2} ConfigArray arrayHetro = arrayTests("arrayHetro"); str = arrayHetro[0]; wstr = (std::wstring) arrayHetro[1]; i = arrayHetro[2]; d = arrayHetro[3]; bt = arrayHetro[4]; ConfigParameters dict(arrayHetro[5]); std::string name = dict.Name(); // arrayNested={|1:2:3|first:second:third|t:f:t|{|identical*2|separator*1|nested*3}} ConfigArray arrayNested = arrayTests(L"arrayNested"); name = arrayNested.Name(); ConfigArray array1 = arrayNested[0]; name = array1.Name(); ConfigArray array2 = arrayNested[1]; name = array2.Name(); ConfigArray array3 = arrayNested[2]; name = array3.Name(); ConfigArray array4 = arrayNested[3]; // // #dictionary tests // dictEmpty=[] ConfigParameters dictEmpty(dictTests("dictEmpty")); // dictSingle=first=1 ConfigParameters dictSingle(dictTests("dictSingle")); // dictMultiple=first=hello;second=there ConfigParameters dictMultiple(dictTests("dictMultiple")); // dictMultipleBraces=[first=hello;second=there;third=with;forth=braces] ConfigParameters dictMultipleBraces(dictTests(L"dictMultipleBraces")); // dictMultiple = dictTests("nothere", dictMultipleBraces); no longer supported, can add if we need // dictMultipleSeparatorBraces=[|first=hello|second=1|thirdBool|forth=12.345|fifth="quoted string"|sixth=古部|seventh=braces] ConfigParameters dictMultipleSeparatorBraces(dictTests("dictMultipleSeparatorBraces")); // dictQuotedStrings=[ // files={ // "c:\path with spaces\file.txt" // "d:\data\Jan 21 1999.my file.txt" // } // mapping="e:\the path\that has\spaces" // ] ConfigParameters dictQuotedStrings(dictTests("dictQuotedStrings")); arrayQuotedStrings = dictQuotedStrings("files"); const char* mapping = dictQuotedStrings("mapping"); mapping; // // #super nesting // dictNested=[ // array={ // dict1=1;dict2=2;dict3=3 // #embedded comment - non-named dictionary // [ // first=1 // second=2 // third=3 // ] // { // [%first=n1%second=n2%third=n3%forth=embedded;old;separator] // d1=n1;d2=n2;d3=n3 // } // } // dict2=[first=1;second=2;third=3] // dict3=[@ // first=1@second=2 // third=3 // forth=4@fifth=5 // ] // ] ConfigParameters dictNested(config("dictNested")); name = dictNested.Name(); ConfigArray array = dictNested("array"); name = array.Name(); ConfigParameters dictElement1(array[0]); name = dictElement1.Name(); ConfigParameters dictElement2(array[1]); name = dictElement2.Name(); ConfigArray arrayNest(array[2]); name = arrayNest.Name(); ConfigParameters dictNElement1(arrayNest[0]); name = dictNElement1.Name(); ConfigParameters dictNElement2(arrayNest[1]); name = dictNElement2.Name(); ConfigParameters dict2(dictNested(L"dict2")); name = dict2.Name(); ConfigParameters dict3(dictNested("dict3")); name = dict3.Name(); //] // File file(L"c:\\temp\\testing.txt", fileOptionsRead | fileOptionsText); // char marker[5]; // if (file.TryGetMarker(fileMarkerBeginSection, L"BVAL")) // { // if (file.TryGetMarker(fileMarkerBeginSection, L"BNEW")) // { // if (!file.TryGetMarker(fileMarkerBeginSection, L"OTHER")) // { // float val; // file >> val; // printf("pass"); // } // } // } // TestConfiguration<ElemType>(configBase); TestMacros<ElemType>(configBase); }