// TODO: This class should go away eventually. // TODO: The composition of packer + randomizer + different deserializers in a generic manner is done in the CompositeDataReader. // TODO: Currently preserving this for backward compatibility with current configs. CNTKTextFormatReader::CNTKTextFormatReader(MemoryProviderPtr provider, const ConfigParameters& config) : m_provider(provider) { TextConfigHelper configHelper(config); try { if (configHelper.GetElementType() == ElementType::tfloat) { m_deserializer = shared_ptr<IDataDeserializer>(new TextParser<float>(configHelper)); } else { m_deserializer = shared_ptr<IDataDeserializer>(new TextParser<double>(configHelper)); } if (configHelper.ShouldKeepDataInMemory()) { m_deserializer = shared_ptr<IDataDeserializer>(new ChunkCache(m_deserializer)); } size_t window = configHelper.GetRandomizationWindow(); if (window > 0) { // Verbosity is a general config parameter, not specific to the text format reader. int verbosity = config(L"verbosity", 0); m_randomizer = make_shared<BlockRandomizer>(verbosity, window, m_deserializer); } else { m_randomizer = std::make_shared<NoRandomizer>(m_deserializer); } if (configHelper.IsInFrameMode()) { m_packer = std::make_shared<FramePacker>( m_provider, m_randomizer, GetStreamDescriptions()); } else { m_packer = std::make_shared<SequencePacker>( m_provider, m_randomizer, GetStreamDescriptions()); } } catch (const std::runtime_error& e) { RuntimeError("CNTKTextFormatReader: While reading '%ls': %s", configHelper.GetFilePath().c_str(), e.what()); } }
void ReaderBase::StartEpoch(const EpochConfiguration& config, const std::map<std::wstring, int>& inputDescriptions) { if (config.m_totalEpochSizeInSamples == 0) { RuntimeError("Epoch size cannot be 0."); } // Let's check that streams requested for this epoch match the ones for previous epochs. // If not, update them. auto streams = GetStreamDescriptions(); if (inputDescriptions.size() != m_requiredInputs.size() || !std::equal(inputDescriptions.begin(), inputDescriptions.end(), m_requiredInputs.begin())) { m_requiredInputs = inputDescriptions; // Reallocating memory providers. m_memoryProviders.resize(streams.size()); for (size_t i = 0; i < streams.size(); ++i) { // TODO: In case when the network requires less inputs, // we should not even have them. if (m_requiredInputs.find(streams[i].m_name) == m_requiredInputs.end()) { m_memoryProviders[i] = std::make_shared<HeapMemoryProvider>(); continue; } int deviceId = m_requiredInputs[streams[i].m_name]; if (deviceId < 0) m_memoryProviders[i] = std::make_shared<HeapMemoryProvider>(); else m_memoryProviders[i] = std::make_shared<CudaMemoryProvider>(deviceId); } } m_sequenceEnumerator->StartEpoch(config); m_packer->SetConfiguration(config, m_memoryProviders); }