TextConfigHelper::TextConfigHelper(const ConfigParameters& config)
{
if (!config.ExistsCurrent(L"input"))
{
RuntimeError("CNTKTextFormatReader configuration does not contain \"input\" section.");
}
const ConfigParameters& input = config(L"input");
if (input.empty())
{
RuntimeError("CNTKTextFormatReader configuration contains an empty \"input\" section.");
}
string precision = config.Find("precision", "float");
if (AreEqualIgnoreCase(precision, "double"))
{
m_elementType = ElementType::tdouble;
}
else if (AreEqualIgnoreCase(precision, "float"))
{
m_elementType = ElementType::tfloat;
}
else
{
RuntimeError("Not supported precision '%s'. Expected 'double' or 'float'.", precision.c_str());
}
StreamId id = 0;
map<string, wstring> aliasToInputMap;
for (const pair<string, ConfigParameters>& section : input)
{
ConfigParameters input = section.second;
wstring name = msra::strfun::utf16(section.first);
if (!input.ExistsCurrent(L"dim") || !input.ExistsCurrent(L"format"))
{
RuntimeError("Input section for input '%ls' does not specify all the required parameters, "
"\"dim\" and \"format\".", name.c_str());
}
StreamDescriptor stream;
stream.m_id = id++;
stream.m_name = name;
stream.m_sampleDimension = input(L"dim");
string type = input(L"format");
if (AreEqualIgnoreCase(type, "dense"))
{
stream.m_storageType = StorageType::dense;
}
else if (AreEqualIgnoreCase(type, "sparse"))
{
stream.m_storageType = StorageType::sparse_csc;
if (stream.m_sampleDimension > numeric_limits<IndexType>::max())
{
RuntimeError("Sample dimension (%" PRIu64 ") for sparse input '%ls'"
" exceeds the maximum allowed value (%" PRIu64 ").\n",
stream.m_sampleDimension, name.c_str(), (size_t)numeric_limits<IndexType>::max());
}
}
else
{
RuntimeError("'format' parameter must be set either to 'dense' or 'sparse'.");
}
// alias is optional
if (input.ExistsCurrent(L"alias"))
{
stream.m_alias = input(L"alias");
if (stream.m_alias.empty())
{
RuntimeError("Alias value for input '%ls' is empty.", name.c_str());
}
}
else
{
stream.m_alias = section.first;
}
if (aliasToInputMap.find(stream.m_alias) != aliasToInputMap.end())
{
RuntimeError("Alias %s is already mapped to input %ls.",
stream.m_alias.c_str(), aliasToInputMap[stream.m_alias].c_str());
}
else
{
aliasToInputMap[stream.m_alias] = stream.m_name;
}
stream.m_elementType = m_elementType;
m_streams.push_back(stream);
}
m_filepath = msra::strfun::utf16(config(L"file"));
if (config.Exists(L"randomize"))
{
wstring randomizeString = config.CanBeString(L"randomize") ? config(L"randomize") : wstring();
if (!_wcsicmp(randomizeString.c_str(), L"none"))
{
m_randomizationWindow = randomizeNone;
}
else if (!_wcsicmp(randomizeString.c_str(), L"auto"))
{
m_randomizationWindow = randomizeAuto;
}
else
{
m_randomizationWindow = config(L"randomize");
}
}
else
{
m_randomizationWindow = randomizeAuto;
}
m_skipSequenceIds = config(L"skipSequenceIds", false);
m_maxErrors = config(L"maxErrors", 0);
m_traceLevel = config(L"traceLevel", 0);
m_chunkSizeBytes = config(L"chunkSizeInBytes", 32 * 1024 * 1024); // 32 MB by default
m_chunkCacheSize = config(L"numChunksToCache", 32); // 32 * 32 MB = 1 GB of memory in total
}
// The whole CompositeDataReader is meant as a stopgap to allow deserializers/transformers composition until SGD talkes
// directly to the new Reader API.
// For more information please see its header file.
// This method composes together packers + randomizer + a set of transformers and deserializers.
CompositeDataReader::CompositeDataReader(const ConfigParameters& config, MemoryProviderPtr provider) : m_layout(make_shared<MBLayout>()),
m_corpus(std::make_shared<CorpusDescriptor>()),
m_provider(provider)
{
wstring action = config(L"action", L"");
bool isActionWrite = AreEqualIgnoreCase(action, L"write");
// Identifying packing mode.
bool frameMode = config(L"frameMode", false);
bool truncated = config(L"truncated", false);
if (frameMode && truncated)
{
LogicError("frameMode and truncated BPTT are mutually exclusive.");
}
if (isActionWrite) // For writing we always use sequence mode.
{
m_packingMode = PackingMode::sequence;
}
else if (frameMode)
{
m_packingMode = PackingMode::sample;
}
else if (truncated)
{
m_packingMode = PackingMode::truncated;
m_truncationLength = config(L"truncationLength", 0);
if (m_truncationLength == 0)
{
InvalidArgument("Truncation length cannot be 0.");
}
}
else
{
m_packingMode = PackingMode::sequence;
}
m_precision = config("precision", "float");
// Creating deserializers.
// TODO: Currently the primary deserializer defines the corpus. The logic will be moved to CorpusDescriptor class.
CreateDeserializers(config);
if (m_deserializers.empty())
{
InvalidArgument("Could not find deserializers in the reader config.");
}
IDataDeserializerPtr deserializer = m_deserializers.front();
if (m_deserializers.size() > 1)
{
// Bundling deserializers together.
// Option whether we need to check data between different deserializers.
bool cleanse = config(L"checkData", true);
deserializer = std::make_shared<Bundler>(config, deserializer, m_deserializers, cleanse);
}
int verbosity = config(L"verbosity", 0);
// Pick up the randomizer, always picking up no randomization for the write mode.
bool randomize = isActionWrite ? false : config(L"randomize", false);
// By default do not use omp threads for deserialization of sequences.
// It makes sense to put it to true for cases when deserialization is CPU intensive,
// i.e. decompression of images.
bool multiThreadedDeserialization = config(L"multiThreadedDeserialization", false);
if (randomize)
{
// By default randomizing the whole data set.
size_t randomizationWindow = config(L"randomizationWindow", requestDataSize);
// By default using STL random number generator.
bool useLegacyRandomization = config(L"useLegacyRandomization", false);
m_sequenceEnumerator = std::make_shared<BlockRandomizer>(verbosity, randomizationWindow, deserializer, BlockRandomizer::DecimationMode::chunk, useLegacyRandomization, multiThreadedDeserialization);
}
else
{
m_sequenceEnumerator = std::make_shared<NoRandomizer>(deserializer, multiThreadedDeserialization);
}
// In case when there are transforms, applying them to the data.
m_sequenceEnumerator = m_transforms.empty()
? m_sequenceEnumerator
: std::make_shared<TransformController>(m_transforms, m_sequenceEnumerator);
// Create output stream descriptions - where to get those? from config? what if it is not the same as network expects?
// TODO: Currently only dense output streams.
// TODO: Check here. We should already support repacking sparse into dense in the shim/matrix.
for (const auto& streamDescription : m_sequenceEnumerator->GetStreamDescriptions())
{
StreamDescriptionPtr stream = std::make_shared<StreamDescription>(*streamDescription);
stream->m_storageType = StorageType::dense;
m_streams.push_back(stream);
m_nameToStreamId.insert(std::make_pair(streamDescription->m_name, streamDescription->m_id));
}
}
ImageConfigHelper::ImageConfigHelper(const ConfigParameters& config)
: m_dataFormat(CHW)
{
std::vector<std::string> featureNames = GetSectionsWithParameter(config, "width");
std::vector<std::string> labelNames = GetSectionsWithParameter(config, "labelDim");
// REVIEW alexeyk: currently support only one feature and label section.
if (featureNames.size() != 1 || labelNames.size() != 1)
{
RuntimeError(
"ImageReader currently supports a single feature and label stream. '%d' features , '%d' labels found.",
static_cast<int>(featureNames.size()),
static_cast<int>(labelNames.size()));
}
ConfigParameters featureSection = config(featureNames[0]);
size_t w = featureSection("width");
size_t h = featureSection("height");
size_t c = featureSection("channels");
std::string mbFmt = featureSection("mbFormat", "nchw");
if (AreEqualIgnoreCase(mbFmt, "nhwc") || AreEqualIgnoreCase(mbFmt, "legacy"))
{
m_dataFormat = HWC;
}
else if (!AreEqualIgnoreCase(mbFmt, "nchw") || AreEqualIgnoreCase(mbFmt, "cudnn"))
{
RuntimeError("ImageReader does not support the sample format '%s', only 'nchw' and 'nhwc' are supported.", mbFmt.c_str());
}
auto features = std::make_shared<StreamDescription>();
features->m_id = 0;
features->m_name = msra::strfun::utf16(featureSection.ConfigName());
features->m_sampleLayout = std::make_shared<TensorShape>(ImageDimensions(w, h, c).AsTensorShape(m_dataFormat));
m_streams.push_back(features);
ConfigParameters label = config(labelNames[0]);
size_t labelDimension = label("labelDim");
auto labelSection = std::make_shared<StreamDescription>();
labelSection->m_id = 1;
labelSection->m_name = msra::strfun::utf16(label.ConfigName());
labelSection->m_sampleLayout = std::make_shared<TensorShape>(labelDimension);
m_streams.push_back(labelSection);
m_mapPath = config(L"file");
std::string rand = config(L"randomize", "auto");
if (AreEqualIgnoreCase(rand, "auto"))
{
m_randomize = true;
}
else if (AreEqualIgnoreCase(rand, "none"))
{
m_randomize = false;
}
else
{
RuntimeError("'randomize' parameter must be set to 'auto' or 'none'");
}
// Identify precision
string precision = config.Find("precision", "float");
if (AreEqualIgnoreCase(precision, "float"))
{
features->m_elementType = ElementType::tfloat;
labelSection->m_elementType = ElementType::tfloat;
}
else if (AreEqualIgnoreCase(precision, "double"))
{
features->m_elementType = ElementType::tdouble;
labelSection->m_elementType = ElementType::tdouble;
}
else
{
RuntimeError("Not supported precision '%s'. Expected 'double' or 'float'.", precision.c_str());
}
m_cpuThreadCount = config(L"numCPUThreads", 0);
}
HTKMLFReader::HTKMLFReader(MemoryProviderPtr provider,
const ConfigParameters& readerConfig)
: m_seed(0), m_provider(provider)
{
// TODO: deserializers and transformers will be dynamically loaded
// from external libraries based on the configuration/brain script.
bool frameMode = readerConfig(L"frameMode", true);
bool truncated = readerConfig(L"truncated", false);
if (frameMode && truncated)
{
LogicError("frameMode and truncated BPTT are mutually exclusive.");
}
if (frameMode)
{
m_packingMode = PackingMode::sample;
}
else if (truncated)
{
m_packingMode = PackingMode::truncated;
}
else
{
m_packingMode = PackingMode::sequence;
}
// nbruttsineachrecurrentiter is old reader configuration, truncationLength is the new one.
// If truncation length is specified we estimate
// the number of parallel sequences we have to pack as max(1, (mbsize/truncationLength))
// If nbruttsineachrecurrentiter is specified we assume that the truncation size is mbSize
// and the real minibatch size in mbSize * nbruttsineachrecurrentiter[epochIndex]
m_truncationLength = readerConfig(L"truncationLength", 0);
m_numParallelSequencesForAllEpochs =
readerConfig(L"nbruttsineachrecurrentiter", ConfigParameters::Array(intargvector(vector<int> { 1 })));
ConfigHelper config(readerConfig);
size_t window = config.GetRandomizationWindow();
auto deserializers = CreateDeserializers(readerConfig);
if (deserializers.empty())
{
LogicError("Please specify at least a single input stream.");
}
auto bundler = std::make_shared<Bundler>(readerConfig, deserializers[0], deserializers, false);
int verbosity = readerConfig(L"verbosity", 2);
std::wstring readMethod = config.GetRandomizer();
// TODO: this should be bool. Change when config per deserializer is allowed.
if (AreEqualIgnoreCase(readMethod, std::wstring(L"blockRandomize")))
{
m_randomizer = std::make_shared<BlockRandomizer>(verbosity, window, bundler, BlockRandomizer::DecimationMode::chunk, true /* useLegacyRandomization */);
}
else if (AreEqualIgnoreCase(readMethod, std::wstring(L"none")))
{
m_randomizer = std::make_shared<NoRandomizer>(bundler);
}
else
{
RuntimeError("readMethod must be 'blockRandomize' or 'none'.");
}
m_randomizer->Initialize(nullptr, readerConfig);
// Create output stream descriptions (all dense)
for (auto d : deserializers)
{
for (auto i : d->GetStreamDescriptions())
{
StreamDescriptionPtr stream = std::make_shared<StreamDescription>(*i);
stream->m_storageType = StorageType::dense;
stream->m_id = m_streams.size();
m_streams.push_back(stream);
}
}
// TODO: should we unify sample and sequence mode packers into a single one.
// TODO: functionally they are the same, the only difference is how we handle
// TODO: MBlayout and what is the perf hit for iterating/copying sequences.
// TODO: Should do more perf tests before unifying these two.
// TODO: As the next step the packers will be moved out of the readers into the
// TODO: core CNTK. They are format agnostic and can be used with any type of
// TODO: deserializers.
switch (m_packingMode)
{
case PackingMode::sample:
m_packer = std::make_shared<FramePacker>(m_provider, m_randomizer, m_streams);
break;
case PackingMode::sequence:
m_packer = std::make_shared<SequencePacker>(m_provider, m_randomizer, m_streams);
break;
case PackingMode::truncated:
m_packer = std::make_shared<TruncatedBPTTPacker>(m_provider, m_randomizer, m_streams);
break;
default:
LogicError("Unsupported type of packer '%d'.", (int)m_packingMode);
}
}
