Base64ImageDeserializer::Base64ImageDeserializer(CorpusDescriptorPtr corpus, const ConfigParameters& config, bool primary) : ImageDeserializerBase(corpus, config, primary)
{
auto mapFile = config(L"file");
bool hasSequenceKeys = HasSequenceKeys(mapFile);
m_fileName.assign(mapFile.begin(), mapFile.end());
attempt(5, [this, hasSequenceKeys, corpus]()
{
if (!m_dataFile || ferror(m_dataFile.get()) != 0)
m_dataFile.reset(fopenOrDie(m_fileName, L"rbS"), [](FILE* f) { if (f) fclose(f); });
m_indexer = make_unique<Indexer>(m_dataFile.get(), m_primary, !hasSequenceKeys);
m_indexer->Build(corpus);
});
}
void TensorBoardFileWriter::Init()
{
time_t time = std::time(0);
std::wstring filePath = GetNewFilePath(m_dir, time);
msra::files::make_intermediate_dirs(filePath);
m_file = fopenOrDie(ToString(filePath), "wb");
m_fileName = filePath;
// Write the first record with the current version, and flush
// right away so the file contents will be easily determined.
WriteVersion(time);
if (m_model)
{
WriteModel();
}
Flush();
}
ImageChunk(const ChunkDescriptor& descriptor, Base64ImageDeserializer& parent)
: m_descriptor(descriptor), m_deserializer(parent)
{
// Let's see if the open descriptor has problems.
if (ferror(m_deserializer.m_dataFile.get()) != 0)
m_deserializer.m_dataFile.reset(fopenOrDie(m_deserializer.m_fileName.c_str(), L"rbS"), [](FILE* f) { if (f) fclose(f); });
if (descriptor.m_sequences.empty() || !descriptor.m_byteSize)
LogicError("Empty chunks are not supported.");
m_buffer.resize(descriptor.m_byteSize + 1);
// Make sure we always have 0 at the end for buffer overrun.
m_buffer[descriptor.m_byteSize] = 0;
m_chunkOffset = descriptor.m_sequences.front().m_fileOffsetBytes;
// Read chunk into memory.
int rc = _fseeki64(m_deserializer.m_dataFile.get(), m_chunkOffset, SEEK_SET);
if (rc)
RuntimeError("Error seeking to position '%" PRId64 "' in the input file '%ls', error code '%d'", m_chunkOffset, m_deserializer.m_fileName.c_str(), rc);
freadOrDie(m_buffer.data(), descriptor.m_byteSize, 1, m_deserializer.m_dataFile.get());
}
// all constructors call this
void File::Init(const wchar_t* filename, int fileOptions)
{
m_filename = filename;
m_options = fileOptions;
if (m_filename.empty())
RuntimeError("File: filename is empty");
const auto outputPipe = (m_filename.front() == '|');
const auto inputPipe = (m_filename.back() == '|');
// translate the options string into a string for fopen()
const auto reading = !!(fileOptions & fileOptionsRead);
const auto writing = !!(fileOptions & fileOptionsWrite);
if (!reading && !writing)
RuntimeError("File: either fileOptionsRead or fileOptionsWrite must be specified");
// convert fileOptions to fopen()'s mode string
wstring options = reading ? L"r" : L"";
if (writing)
{
// if we already are reading the file, change to read/write
options.clear();
options.append(L"w");
if (!outputPipe && m_filename != L"-")
{
options.append(L"+");
msra::files::make_intermediate_dirs(m_filename.c_str()); // writing to regular file -> also create the intermediate directories as a convenience
}
}
if (fileOptions & fileOptionsBinary)
options += L"b";
else
options += L"t";
// add sequential flag to allocate big read buffer
if (fileOptions & fileOptionsSequential)
options += L"S";
// now open the file
// Special path syntax understood here:
// - "-" refers to stdin or stdout
// - "|cmd" writes to a pipe
// - "cmd|" reads from a pipe
m_pcloseNeeded = false;
m_seekable = false;
if (m_filename == L"-") // stdin/stdout
{
if (writing && reading)
RuntimeError("File: cannot specify fileOptionsRead and fileOptionsWrite at once with path '-'");
m_file = writing ? stdout : stdin;
}
else if (outputPipe || inputPipe) // pipe syntax
{
if (inputPipe && outputPipe)
RuntimeError("File: pipes cannot specify fileOptionsRead and fileOptionsWrite at once");
if (inputPipe != reading)
RuntimeError("File: pipes must use consistent fileOptionsRead/fileOptionsWrite");
const auto command = inputPipe ? m_filename.substr(0, m_filename.size() - 1) : m_filename.substr(1);
m_file = _wpopen(command.c_str(), options.c_str());
if (!m_file)
RuntimeError("File: error exexuting pipe command '%S': %s", command.c_str(), strerror(errno));
m_pcloseNeeded = true;
}
else
attempt([=]() // regular file: use a retry loop
{
m_file = fopenOrDie(filename, options.c_str());
m_seekable = true;
});
}
