void CropTransformer::InitFromConfig(const ConfigParameters &config)
{
m_cropType = ParseCropType(config(L"cropType", ""));
floatargvector cropRatio = config(L"cropRatio", "1.0");
m_cropRatioMin = cropRatio[0];
m_cropRatioMax = cropRatio[1];
if (!(0 < m_cropRatioMin && m_cropRatioMin <= 1.0) ||
!(0 < m_cropRatioMax && m_cropRatioMax <= 1.0) ||
m_cropRatioMin > m_cropRatioMax)
{
RuntimeError("Invalid cropRatio value, must be > 0 and <= 1. cropMin must "
"<= cropMax");
}
m_jitterType = ParseJitterType(config(L"jitterType", ""));
if (!config.ExistsCurrent(L"hflip"))
{
m_hFlip = m_cropType == CropType::Random;
}
else
{
m_hFlip = config(L"hflip");
}
}
void CropTransformer::InitFromConfig(const ConfigParameters &config)
{
floatargvector cropRatio = config(L"cropRatio", "1.0");
m_cropRatioMin = cropRatio[0];
m_cropRatioMax = cropRatio[1];
if (!(0 < m_cropRatioMin && m_cropRatioMin <= 1.0) ||
!(0 < m_cropRatioMax && m_cropRatioMax <= 1.0) ||
m_cropRatioMin > m_cropRatioMax)
{
RuntimeError("Invalid cropRatio value, must be > 0 and <= 1. cropMin must "
"<= cropMax");
}
m_jitterType = ParseJitterType(config(L"jitterType", ""));
if (!config.ExistsCurrent(L"hflip"))
{
m_hFlip = m_imageConfig->GetCropType() == CropType::Random;
}
else
{
m_hFlip = config(L"hflip");
}
m_aspectRatioRadius = config(L"aspectRatioRadius", ConfigParameters::Array(doubleargvector(vector<double>{0.0})));
}
// special temporary function to guard against a now invalid usage of "truncated" which exists in some IPG production setups
static void DisableLegacyTruncationSettings(const ConfigParameters& TopLevelConfig, const ConfigParameters& commandConfig)
{
if (TopLevelConfig.ExistsCurrent(L"Truncated"))
{
return;
}
// if any of the action has set a reader/SGD section and has different Truncated value for reader and SGD section
ConfigArray actions = commandConfig(L"action");
for (size_t i = 0; i < actions.size(); i++)
{
if (actions[i] == "train" || actions[i] == "trainRNN")
{
ConfigParameters sgd = ConfigParameters(commandConfig(L"SGD"));
ConfigParameters reader = ConfigParameters(commandConfig(L"reader"));
// reader and SGD sections are two must-have sections in train/trainRNN
if (reader.ExistsCurrent(L"Truncated") && !sgd.ExistsCurrent(L"Truncated"))
{
InvalidArgument("DisableLegacyUsage: setting Truncated only in reader section are not allowed. Please move Truncated=true/false to the top level section.");
}
}
}
}
// The whole CompositeDataReader is meant as a stopgap to allow deserializers/transformers composition until SGD talkes
// directly to the new Reader API.
// For more information please see its header file.
// This method composes together packers + randomizer + a set of transformers and deserializers.
CompositeDataReader::CompositeDataReader(const ConfigParameters& config) :
m_truncationLength(0)
{
wstring action = config(L"action", L"");
bool isActionWrite = AreEqualIgnoreCase(action, L"write");
// By default, we use numeric sequence keys (i.e., for cbf, ctf, image and base64 readers).
// For MLF and HTK deserializers, we use non-numeric (string) sequence keys.
bool useNumericSequenceKeys = true;
if (ContainsDeserializer(config, L"HTKFeatureDeserializer") ||
ContainsDeserializer(config, L"HTKMLFDeserializer"))
{
useNumericSequenceKeys = false;
}
useNumericSequenceKeys = config(L"useNumericSequenceKeys", useNumericSequenceKeys);
bool useHash = config(L"hashSequenceKeys", false);
m_corpus = std::make_shared<CorpusDescriptor>(useNumericSequenceKeys, useHash);
// Identifying packing mode.
bool frameMode = config(L"frameMode", false);
bool truncated = config(L"truncated", false);
if (frameMode && truncated)
{
LogicError("frameMode and truncated BPTT are mutually exclusive.");
}
if (isActionWrite) // For writing we always use sequence mode.
{
m_packingMode = PackingMode::sequence;
}
else if (frameMode)
{
m_packingMode = PackingMode::sample;
}
else if (truncated)
{
m_packingMode = PackingMode::truncated;
m_truncationLength = config(L"truncationLength", 0);
if (m_truncationLength == 0)
{
InvalidArgument("Truncation length cannot be 0.");
}
}
else
{
m_packingMode = PackingMode::sequence;
}
m_rightSplice = config(L"rightSplice", 0);
if (m_rightSplice > m_truncationLength)
InvalidArgument("rightSplice should not be greater than truncation length");
m_precision = config("precision", "float");
// Creating deserializers.
bool composable = CreateDeserializers(config);
if (m_deserializers.empty())
InvalidArgument("Could not find deserializers in the reader config.");
if (!composable && m_deserializers.size() > 1)
InvalidArgument("Currently user defined deserializers do not support composability. Please specify a single deserializer.");
DataDeserializerPtr deserializer = m_deserializers.front();
if (m_deserializers.size() > 1)
{
// Bundling deserializers together.
// Option whether we need to check data between different deserializers.
bool cleanse = config(L"checkData", true);
deserializer = std::make_shared<Bundler>(config, m_corpus, deserializer, m_deserializers, cleanse);
}
int verbosity = config(L"verbosity", 0);
// Pick up the randomizer, always picking up no randomization for the write mode.
bool randomize = isActionWrite ? false : config(L"randomize", true);
// Get maximum number of allowed errors per worker.
size_t maxErrors = config(L"maxErrors", 0);
// By default do not use omp threads for deserialization of sequences.
// It makes sense to put it to true for cases when deserialization is CPU intensive,
// i.e. decompression of images.
bool multiThreadedDeserialization = config(L"multiThreadedDeserialization", ContainsDeserializer(config, L"ImageDeserializer"));
if (!composable) // Pick up simple interface.
{
if (randomize)
{
bool sampleBasedRandomizationWindow = config(L"sampleBasedRandomizationWindow", false);
m_sequenceEnumerator = std::make_shared<LTTumblingWindowRandomizer>(deserializer,
sampleBasedRandomizationWindow, config(L"randomizationWindow", requestDataSize),
GetRandomSeed(config),
multiThreadedDeserialization, maxErrors);
}
else
m_sequenceEnumerator = std::make_shared<LTNoRandomizer>(deserializer, multiThreadedDeserialization, maxErrors);
}
else
{
if (randomize)
{
// By default randomizing the whole data set.
size_t randomizationWindow = requestDataSize;
// Currently in case of images, a single chunk is a single image. So no need to randomize, chunks will be randomized anyway.
if (ContainsDeserializer(config, L"ImageDeserializer") && m_deserializers.size() == 1)
{
randomizationWindow = 1;
m_packingMode = PackingMode::sample;
}
randomizationWindow = config(L"randomizationWindow", randomizationWindow);
bool sampleBasedRandomizationWindow = config(L"sampleBasedRandomizationWindow", true);
if (ContainsDeserializer(config, L"CNTKTextFormatDeserializer") && !config.ExistsCurrent(L"randomizationWindow"))
{
if (!config.ExistsCurrent(L"sampleBasedRandomizationWindow") || // sampleBasedRandomizationWindow is not specified
!sampleBasedRandomizationWindow) // randomization window is in chunks
{
sampleBasedRandomizationWindow = false;
size_t chunkSizeBytes = config(L"chunkSizeInBytes", g_32MB); // 32 MB by default
randomizationWindow = g_4GB / chunkSizeBytes; // ~ 4 GB disk space worth of chunks
// TODO: decrease randomization window if m_deserializers.size() > 1 ?
}
else
{
// config explicitly says to use a sample-based window, but does not specify its size.
LogicError("'sampleBasedRandomizationWindow' (== 'true') requires that the 'randomizationWindow' is explicitly specified.");
}
}
bool shouldPrefetch = true;
m_sequenceEnumerator = std::make_shared<BlockRandomizer>(verbosity, randomizationWindow, deserializer, shouldPrefetch,
multiThreadedDeserialization, maxErrors, sampleBasedRandomizationWindow, GetRandomSeed(config));
}
else
m_sequenceEnumerator = std::make_shared<NoRandomizer>(deserializer, multiThreadedDeserialization, maxErrors);
}
// In case when there are transforms, applying them to the data.
m_sequenceEnumerator = m_transforms.empty()
? m_sequenceEnumerator
: std::make_shared<TransformController>(m_transforms, m_sequenceEnumerator, multiThreadedDeserialization);
// TODO: Output stream descriptions - this should come from the network so that we can check
// that input matches what the network expects (including tensor shape, etc.).
std::vector<StreamInformation> outputStreams = m_sequenceEnumerator->GetStreamDescriptions();
// Currently for prefetch we use two alternating buffers,
// same is the default.
size_t numAlternatingBuffers = 2;
// Check whether to use local timeline, by default we use it for better performance.
bool localTimeline = config(L"localTimeline", true);
switch (m_packingMode)
{
case PackingMode::sample:
m_packer = std::make_shared<FramePacker>(
m_sequenceEnumerator,
outputStreams,
numAlternatingBuffers,
localTimeline,
m_corpus);
break;
case PackingMode::sequence:
m_packer = std::make_shared<SequencePacker>(
m_sequenceEnumerator,
outputStreams,
numAlternatingBuffers,
localTimeline,
m_corpus);
break;
case PackingMode::truncated:
{
// Currently BPTT does not support sparse format as output.
// We always require dense from the packer.
for (auto& s : outputStreams)
s.m_storageFormat = StorageFormat::Dense;
m_packer = std::make_shared<TruncatedBPTTPacker>(
m_sequenceEnumerator,
outputStreams,
numAlternatingBuffers,
m_corpus);
break;
}
default:
LogicError("Unsupported type of packer '%d'.", (int)m_packingMode);
}
}
CropTransformer::CropTransformer(const ConfigParameters& config) : ImageTransformerBase(config)
{
intargvector cropSize = config(L"cropSize", "0");
m_cropWidth = cropSize[0];
m_cropHeight = cropSize[1];
if (m_cropWidth < 0 || m_cropHeight < 0)
{
RuntimeError("Invalid cropSize value, must be >= 0");
}
m_useSideRatio = true;
floatargvector sideRatio = config(L"sideRatio", "0.0");
m_sideRatioMin = sideRatio[0];
m_sideRatioMax = sideRatio[1];
if (m_sideRatioMin == 0.0 && m_sideRatioMax == 0.0) // taking default value means not specified
{
m_useSideRatio = false;
}
else if (!(m_sideRatioMin > 0 && m_sideRatioMax <= 1.0) ||
m_sideRatioMin > m_sideRatioMax)
{
RuntimeError("Invalid sideRatio value, must be > 0 and <= 1. sideMin must <= sideMax");
}
m_useAreaRatio = true;
floatargvector areaRatio = config(L"areaRatio", "0.0");
m_areaRatioMin = areaRatio[0];
m_areaRatioMax = areaRatio[1];
if (m_areaRatioMin == 0.0 && m_areaRatioMax == 0.0) // taking default value means not specified
{
m_useAreaRatio = false;
}
else if (!(m_areaRatioMin > 0 && m_areaRatioMax <= 1.0) ||
m_areaRatioMin > m_areaRatioMax)
{
RuntimeError("Invalid areaRatio value, must be > 0 and <= 1. areaMin must <= areaMax");
}
if (m_useSideRatio && m_useAreaRatio)
RuntimeError("sideRatio and areaRatio cannot be specified simultaneously");
floatargvector aspectRatio = config(L"aspectRatio", "1.0");
m_aspectRatioMin = aspectRatio[0];
m_aspectRatioMax = aspectRatio[1];
if (!(m_aspectRatioMin > 0 && m_aspectRatioMax <= 1.0) ||
m_aspectRatioMin > m_aspectRatioMax)
{
RuntimeError("Invalid aspectRatio value, must be > 0 and <= 1. aspectMin must <= aspectMax");
}
m_jitterType = ParseJitterType(config(L"jitterType", ""));
m_cropType = ImageConfigHelper::ParseCropType(config(L"cropType", ""));
if (!config.ExistsCurrent(L"hflip"))
{
m_hFlip = (m_cropType == CropType::RandomSide || m_cropType == CropType::RandomArea);
}
else
{
m_hFlip = config(L"hflip");
}
// for MultiView10 we need to set m_hflip = false, otherwise we might not get 5 unflipped image (see CropTransformer::Apply below)
if (m_cropType == CropType::MultiView10)
{
m_hFlip = false;
}
}
TextConfigHelper::TextConfigHelper(const ConfigParameters& config)
{
if (!config.ExistsCurrent(L"input"))
{
RuntimeError("CNTKTextFormatReader configuration does not contain \"input\" section.");
}
const ConfigParameters& input = config(L"input");
if (input.empty())
{
RuntimeError("CNTKTextFormatReader configuration contains an empty \"input\" section.");
}
string precision = config.Find("precision", "float");
if (AreEqualIgnoreCase(precision, "double"))
{
m_elementType = ElementType::tdouble;
}
else if (AreEqualIgnoreCase(precision, "float"))
{
m_elementType = ElementType::tfloat;
}
else
{
RuntimeError("Not supported precision '%s'. Expected 'double' or 'float'.", precision.c_str());
}
StreamId id = 0;
map<string, wstring> aliasToInputMap;
for (const pair<string, ConfigParameters>& section : input)
{
ConfigParameters input = section.second;
wstring name = msra::strfun::utf16(section.first);
if (!input.ExistsCurrent(L"dim") || !input.ExistsCurrent(L"format"))
{
RuntimeError("Input section for input '%ls' does not specify all the required parameters, "
"\"dim\" and \"format\".", name.c_str());
}
StreamDescriptor stream;
stream.m_id = id++;
stream.m_name = name;
stream.m_sampleDimension = input(L"dim");
string type = input(L"format");
if (AreEqualIgnoreCase(type, "dense"))
{
stream.m_storageType = StorageType::dense;
}
else if (AreEqualIgnoreCase(type, "sparse"))
{
stream.m_storageType = StorageType::sparse_csc;
if (stream.m_sampleDimension > numeric_limits<IndexType>::max())
{
RuntimeError("Sample dimension (%" PRIu64 ") for sparse input '%ls'"
" exceeds the maximum allowed value (%" PRIu64 ").\n",
stream.m_sampleDimension, name.c_str(), (size_t)numeric_limits<IndexType>::max());
}
}
else
{
RuntimeError("'format' parameter must be set either to 'dense' or 'sparse'.");
}
// alias is optional
if (input.ExistsCurrent(L"alias"))
{
stream.m_alias = input(L"alias");
if (stream.m_alias.empty())
{
RuntimeError("Alias value for input '%ls' is empty.", name.c_str());
}
}
else
{
stream.m_alias = section.first;
}
if (aliasToInputMap.find(stream.m_alias) != aliasToInputMap.end())
{
RuntimeError("Alias %s is already mapped to input %ls.",
stream.m_alias.c_str(), aliasToInputMap[stream.m_alias].c_str());
}
else
{
aliasToInputMap[stream.m_alias] = stream.m_name;
}
stream.m_elementType = m_elementType;
m_streams.push_back(stream);
}
m_filepath = msra::strfun::utf16(config(L"file"));
if (config.Exists(L"randomize"))
{
wstring randomizeString = config.CanBeString(L"randomize") ? config(L"randomize") : wstring();
if (!_wcsicmp(randomizeString.c_str(), L"none"))
{
m_randomizationWindow = randomizeNone;
}
else if (!_wcsicmp(randomizeString.c_str(), L"auto"))
{
m_randomizationWindow = randomizeAuto;
}
else
{
m_randomizationWindow = config(L"randomize");
}
}
else
{
m_randomizationWindow = randomizeAuto;
}
m_skipSequenceIds = config(L"skipSequenceIds", false);
m_maxErrors = config(L"maxErrors", 0);
m_traceLevel = config(L"traceLevel", 0);
m_chunkSizeBytes = config(L"chunkSizeInBytes", 32 * 1024 * 1024); // 32 MB by default
m_chunkCacheSize = config(L"numChunksToCache", 32); // 32 * 32 MB = 1 GB of memory in total
}
TextConfigHelper::TextConfigHelper(const ConfigParameters& config)
{
if (!config.ExistsCurrent(L"input"))
{
RuntimeError("CNTKTextFormatReader configuration does not contain \"input\" section.");
}
const ConfigParameters& input = config(L"input");
if (input.empty())
{
RuntimeError("CNTKTextFormatReader configuration contains an empty \"input\" section.");
}
string precision = config.Find("precision", "float");
if (AreEqualIgnoreCase(precision, "double"))
{
m_elementType = DataType::Double;
}
else if (AreEqualIgnoreCase(precision, "float"))
{
m_elementType = DataType::Float;
}
else
{
RuntimeError("Not supported precision '%s'. Expected 'double' or 'float'.", precision.c_str());
}
StreamId id = 0;
map<string, wstring> aliasToInputMap;
for (const pair<string, ConfigParameters>& section : input)
{
ConfigParameters input2 = section.second;
wstring name = msra::strfun::utf16(section.first);
if (!input2.ExistsCurrent(L"dim") || !input2.ExistsCurrent(L"format"))
{
RuntimeError("Input section for input '%ls' does not specify all the required parameters, "
"\"dim\" and \"format\".", name.c_str());
}
StreamDescriptor stream;
stream.m_id = id++;
stream.m_name = name;
stream.m_sampleDimension = input2(L"dim");
stream.m_definesMbSize = input2(L"definesMBSize", false);
string type = input2(L"format");
if (AreEqualIgnoreCase(type, "dense"))
{
stream.m_storageFormat = StorageFormat::Dense;
}
else if (AreEqualIgnoreCase(type, "sparse"))
{
stream.m_storageFormat = StorageFormat::SparseCSC;
if (stream.m_sampleDimension > numeric_limits<IndexType>::max())
{
RuntimeError("Sample dimension (%" PRIu64 ") for sparse input '%ls'"
" exceeds the maximum allowed value (%" PRIu64 ").\n",
stream.m_sampleDimension, name.c_str(), (size_t)numeric_limits<IndexType>::max());
}
}
else
{
RuntimeError("'format' parameter must be set either to 'dense' or 'sparse'.");
}
// alias is optional
if (input2.ExistsCurrent(L"alias"))
{
stream.m_alias = input2(L"alias");
if (stream.m_alias.empty())
{
RuntimeError("Alias value for input '%ls' is empty.", name.c_str());
}
}
else
{
stream.m_alias = section.first;
}
if (aliasToInputMap.find(stream.m_alias) != aliasToInputMap.end())
{
RuntimeError("Alias %s is already mapped to input %ls.",
stream.m_alias.c_str(), aliasToInputMap[stream.m_alias].c_str());
}
else
{
aliasToInputMap[stream.m_alias] = stream.m_name;
}
stream.m_elementType = m_elementType;
m_streams.push_back(stream);
}
m_filepath = msra::strfun::utf16(config(L"file"));
m_skipSequenceIds = config(L"skipSequenceIds", false);
m_maxErrors = config(L"maxErrors", 0);
m_traceLevel = config(L"traceLevel", 1);
m_chunkSizeBytes = config(L"chunkSizeInBytes", g_32MB); // 32 MB by default
m_keepDataInMemory = config(L"keepDataInMemory", false);
m_frameMode = config(L"frameMode", false);
m_randomizationWindow = GetRandomizationWindowFromConfig(config);
m_sampleBasedRandomizationWindow = config(L"sampleBasedRandomizationWindow", false);
if (!m_sampleBasedRandomizationWindow && m_randomizationWindow == randomizeAuto)
{
m_randomizationWindow = g_4GB / m_chunkSizeBytes; // ~ 4 GB (on disk) worth of chunks
}
}
