int wmainOldCNTKConfig(int argc, wchar_t* argv[]) // called from wmain which is a wrapper that catches & repots Win32 exceptions
{
ConfigParameters config;
std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config);
// get the command param set they want
wstring logpath = config(L"stderr", L"");
// [1/26/2015 erw, add done file so that it can be used on HPC]
wstring DoneFile = config(L"DoneFile", L"");
ConfigArray command = config(L"command", "train");
// paralleltrain training
g_mpi = nullptr;
bool paralleltrain = config(L"parallelTrain", "false");
if (paralleltrain)
{
g_mpi = new MPIWrapper();
}
g_shareNodeValueMatrices = config(L"shareNodeValueMatrices", false);
TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0));
if (logpath != L"")
{
for (int i = 0; i < command.size(); i++)
{
logpath += L"_";
logpath += (wstring) command[i];
}
logpath += L".log";
if (paralleltrain)
{
std::wostringstream oss;
oss << g_mpi->CurrentNodeRank();
logpath += L"rank" + oss.str();
}
RedirectStdErr(logpath);
}
PrintBuiltInfo(); // this one goes to log file
std::string timestamp = TimeDateStamp();
// dump config info
fprintf(stderr, "running on %s at %s\n", GetHostName().c_str(), timestamp.c_str());
fprintf(stderr, "command line: \n");
for (int i = 0; i < argc; i++)
{
fprintf(stderr, "%s ", WCharToString(argv[i]).c_str());
}
// This simply merges all the different config parameters specified (eg, via config files or via command line directly),
// and prints it.
fprintf(stderr, "\n\n>>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>>\n");
fprintf(stderr, "%s\n", rawConfigString.c_str());
fprintf(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<<\n");
// Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overriden at command line),
// All of these assignments will appear, even though only the last assignment matters.
fprintf(stderr, "\n>>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n");
fprintf(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str());
fprintf(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n");
// This outputs the final value each variable/parameter is assigned to in config (so if a parameter is set multiple times, only the last
// value it is set to will appear).
fprintf(stderr, "\n>>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n");
config.dumpWithResolvedVariables();
fprintf(stderr, "<<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n");
fprintf(stderr, "command: ");
for (int i = 0; i < command.size(); i++)
{
fprintf(stderr, "%s ", command[i].c_str());
}
// run commands
std::string type = config(L"precision", "float");
// accept old precision key for backward compatibility
if (config.Exists("type"))
{
type = config(L"type", "float");
}
fprintf(stderr, "\nprecision = %s\n", type.c_str());
if (type == "float")
{
DoCommands<float>(config);
}
else if (type == "double")
{
DoCommands<double>(config);
}
else
{
RuntimeError("invalid precision specified: %s", type.c_str());
}
// still here , write a DoneFile if necessary
if (!DoneFile.empty())
{
FILE* fp = fopenOrDie(DoneFile.c_str(), L"w");
fprintf(fp, "successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str());
fcloseOrDie(fp);
}
fprintf(stderr, "COMPLETED\n"), fflush(stderr);
delete g_mpi;
return EXIT_SUCCESS;
}
// called from wmain which is a wrapper that catches & repots Win32 exceptions
int wmainOldCNTKConfig(int argc, wchar_t* argv[])
{
std::string timestamp = TimeDateStamp();
PrintBanner(argc, argv, timestamp);
ConfigParameters config;
std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config); // get the command param set they want
int traceLevel = config(L"traceLevel", 0);
#ifndef CPUONLY
ConfigValue val = config("deviceId", "auto");
if (!EqualCI(val, "cpu") && !EqualCI(val, "auto"))
{
if (static_cast<int>(val) >= 0) // gpu (id >= 0)
{
CheckSupportForGpu(static_cast<int>(val)); // throws if gpu is not supported
}
}
#endif
if (config(L"timestamping", false))
ProgressTracing::SetTimestampingFlag();
if (config(L"forceDeterministicAlgorithms", false))
Globals::ForceDeterministicAlgorithms();
// get the command param set they want
wstring logpath = config(L"stderr", L"");
wstring doneFile = config(L"doneFile", L"");
ConfigArray command = config(L"command", "train");
// parallel training
// The top-level 'parallelTrain' is a bool, not to be confused with the parallelTrain block inside SGD.
shared_ptr<Microsoft::MSR::CNTK::MPIWrapper> mpi;
auto ensureMPIWrapperCleanup = MakeScopeExit(&MPIWrapper::DeleteInstance);
// when running under MPI with more than one node, use 'true' as the default value for parallelTrain,
// 'false' otherwise.
bool paralleltrain = config(L"parallelTrain", (MPIWrapper::GetTotalNumberOfMPINodes() > 1));
if (paralleltrain)
{
mpi = MPIWrapper::GetInstance(true /*create*/);
}
g_shareNodeValueMatrices = config(L"shareNodeValueMatrices", false);
TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0));
if (logpath != L"")
{
#if 1 // keep the ability to do it how it was done before 1.8; delete if noone needs it anymore
let useOldWay = ProgressTracing::GetTimestampingFlag(); // enable it when running in our server farm
if (useOldWay)
{
for (int i = 0; i < command.size(); i++) // append all 'command' entries
{
logpath += L"_";
logpath += (wstring)command[i];
}
logpath += L".log"; // append .log
}
if (paralleltrain && useOldWay)
{
std::wostringstream oss;
oss << mpi->CurrentNodeRank();
logpath += L"rank" + oss.str();
}
else
#endif
// for MPI workers except main, append .rankN
if (paralleltrain && mpi->CurrentNodeRank() != 0)
logpath += msra::strfun::wstrprintf(L".rank%d", mpi->CurrentNodeRank());
RedirectStdErr(logpath);
if (traceLevel == 0)
PrintBanner(argc, argv, timestamp); // repeat simple banner into log file
}
// full config info
if (traceLevel > 0)
{
PrintBuiltInfo();
PrintGpuInfo();
}
#ifdef _DEBUG
if (traceLevel > 0)
{
// This simply merges all the different config parameters specified (eg, via config files or via command line directly),
// and prints it.
fprintf(stderr, "\nConfiguration, Raw:\n\n");
LOGPRINTF(stderr, "%s\n", rawConfigString.c_str());
// Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overridden at command line),
// All of these assignments will appear, even though only the last assignment matters.
fprintf(stderr, "\nConfiguration After Variable Resolution:\n\n");
LOGPRINTF(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str());
}
#endif
SetMathLibTraceLevel(traceLevel);
// This outputs the final value each variable/parameter is assigned to in config (so if a parameter is set multiple times, only the last
// value it is set to will appear).
if (traceLevel > 0)
{
fprintf(stderr, "\nConfiguration After Processing and Variable Resolution:\n\n");
config.dumpWithResolvedVariables();
LOGPRINTF(stderr, "Commands:");
for (int i = 0; i < command.size(); i++)
fprintf(stderr, " %s", command[i].c_str());
fprintf(stderr, "\n");
}
// run commands
std::string type = config(L"precision", "float");
// accept old precision key for backward compatibility
if (config.Exists("type"))
InvalidArgument("CNTK: Use of 'type' parameter is deprecated, it is called 'precision' now.");
if (traceLevel > 0)
{
LOGPRINTF(stderr, "precision = \"%s\"\n", type.c_str());
}
if (type == "float")
DoCommands<float>(config, mpi);
else if (type == "double")
DoCommands<double>(config, mpi);
else
RuntimeError("CNTK: Invalid precision string: \"%s\", must be \"float\" or \"double\"", type.c_str());
// if completed then write a doneFile if requested
if (!doneFile.empty())
{
FILE* fp = fopenOrDie(doneFile.c_str(), L"w");
fprintf(fp, "Successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str());
fcloseOrDie(fp);
}
if (ProgressTracing::GetTimestampingFlag())
{
LOGPRINTF(stderr, "__COMPLETED__\n"); // running in server environment which expects this string
}
else
fprintf(stderr, "COMPLETED.\n");
fflush(stderr);
return EXIT_SUCCESS;
}
// ---------------------------------------------------------------------------
// main() for old CNTK config language
// ---------------------------------------------------------------------------
// called from wmain which is a wrapper that catches & repots Win32 exceptions
int wmainOldCNTKConfig(int argc, wchar_t* argv[])
{
ConfigParameters config;
std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config); // get the command param set they want
bool timestamping = config(L"timestamping", false);
if (timestamping)
{
ProgressTracing::SetTimestampingFlag();
}
// get the command param set they want
wstring logpath = config(L"stderr", L"");
// [1/26/2015 erw, add done file so that it can be used on HPC]
wstring DoneFile = config(L"DoneFile", L"");
ConfigArray command = config(L"command", "train");
// paralleltrain training
shared_ptr<Microsoft::MSR::CNTK::MPIWrapper> mpi;
bool paralleltrain = config(L"parallelTrain", "false");
if (paralleltrain)
mpi = MPIWrapper::GetInstance(true /*create*/);
g_shareNodeValueMatrices = config(L"shareNodeValueMatrices", false);
TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0));
if (logpath != L"")
{
for (int i = 0; i < command.size(); i++)
{
logpath += L"_";
logpath += (wstring) command[i];
}
logpath += L".log";
if (paralleltrain)
{
std::wostringstream oss;
oss << mpi->CurrentNodeRank();
logpath += L"rank" + oss.str();
}
RedirectStdErr(logpath);
}
PrintBuiltInfo(); // this one goes to log file
std::string timestamp = TimeDateStamp();
// dump config info
fprintf(stderr, "\n");
LOGPRINTF(stderr, "Running on %s at %s\n", GetHostName().c_str(), timestamp.c_str());
LOGPRINTF(stderr, "Command line: \n");
for (int i = 0; i < argc; i++)
fprintf(stderr, "%*s%ls", i > 0 ? 2 : 0, "", argv[i]); // use 2 spaces for better visual separability
fprintf(stderr, "\n\n");
#if 1 //def _DEBUG
// This simply merges all the different config parameters specified (eg, via config files or via command line directly),
// and prints it.
fprintf(stderr, "\n\n");
LOGPRINTF(stderr, ">>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>>\n");
LOGPRINTF(stderr, "%s\n", rawConfigString.c_str());
LOGPRINTF(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<<\n");
// Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overridden at command line),
// All of these assignments will appear, even though only the last assignment matters.
fprintf(stderr, "\n");
LOGPRINTF(stderr, ">>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n");
LOGPRINTF(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str());
LOGPRINTF(stderr, "<<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n");
// This outputs the final value each variable/parameter is assigned to in config (so if a parameter is set multiple times, only the last
// value it is set to will appear).
fprintf(stderr, "\n");
LOGPRINTF(stderr, ">>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>\n");
config.dumpWithResolvedVariables();
LOGPRINTF(stderr, "<<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<\n");
#endif
LOGPRINTF(stderr, "Commands:");
for (int i = 0; i < command.size(); i++)
fprintf(stderr, " %s", command[i].c_str());
fprintf(stderr, "\n");
// run commands
std::string type = config(L"precision", "float");
// accept old precision key for backward compatibility
if (config.Exists("type"))
InvalidArgument("CNTK: Use of 'type' parameter is deprecated, it is called 'precision' now.");
LOGPRINTF(stderr, "Precision = \"%s\"\n", type.c_str());
if (type == "float")
DoCommands<float>(config, mpi);
else if (type == "double")
DoCommands<double>(config, mpi);
else
RuntimeError("CNTK: Invalid precision string: \"%s\", must be \"float\" or \"double\"", type.c_str());
// if completed then write a DoneFile if requested
if (!DoneFile.empty())
{
FILE* fp = fopenOrDie(DoneFile.c_str(), L"w");
fprintf(fp, "successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str());
fcloseOrDie(fp);
}
// TODO: Change back to COMPLETED (no underscores)
LOGPRINTF(stderr, "__COMPLETED__\n");
fflush(stderr);
MPIWrapper::DeleteInstance();
return EXIT_SUCCESS;
}
