virtual void done()
{
CDiskReadMasterVF::done();
IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *)queryHelper();
if (0 != (helper->getFlags() & TDXtemporary) && !container.queryJob().queryUseCheckpoints())
container.queryTempHandler()->deregisterFile(fileName, fileDesc->queryProperties().getPropBool("@pausefile"));
}
void CDiskReadMasterBase::init()
{
IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *) queryHelper();
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), helper->getFileName(), 0 != ((TDXtemporary|TDXjobtemp) & helper->getFlags()), 0 != (TDRoptional & helper->getFlags()), true);
if (file)
{
if (file->numParts() > 1)
fileDesc.setown(getConfiguredFileDescriptor(*file));
else
fileDesc.setown(file->getFileDescriptor());
if (container.queryLocal() || helper->canMatchAny()) // if local, assume may match
{
bool local;
if (0 == (TDXtemporary & helper->getFlags())) // don't add temp files
{
queryThorFileManager().noteFileRead(container.queryJob(), file);
local = container.queryLocal();
}
else
local = false;
mapping.setown(getFileSlaveMaps(file->queryLogicalName(), *fileDesc, container.queryJob().queryUserDescriptor(), container.queryJob().querySlaveGroup(), local, false, hash, file->querySuperFile()));
}
if (0 != (helper->getFlags() & TDRfilenamecallback)) // only get/serialize if using virtual file name fields
{
IDistributedSuperFile *super = file->querySuperFile();
if (super)
{
unsigned numsubs = super->numSubFiles(true);
unsigned s=0;
for (; s<numsubs; s++)
{
IDistributedFile &subfile = super->querySubFile(s, true);
subfileLogicalFilenames.append(subfile.queryLogicalName());
}
}
}
validateFile(file);
void *ekey;
size32_t ekeylen;
helper->getEncryptKey(ekeylen,ekey);
bool encrypted = fileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", helper->getFileName());
container.queryJob().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", helper->getFileName());
}
}
void CDiskReadMasterBase::done()
{
IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *) queryHelper();
fileDesc.clear();
if (!abortSoon) // in case query has relinquished control of file usage to another query (e.g. perists)
{
if (0 != (helper->getFlags() & TDXupdateaccessed))
queryThorFileManager().updateAccessTime(container.queryJob(), helper->getFileName());
}
}
void CDiskReadMasterBase::serializeSlaveData(MemoryBuffer &dst, unsigned slave)
{
IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *) queryHelper();
dst.append(helper->getFileName());
dst.append(subfileLogicalFilenames.ordinality());
if (subfileLogicalFilenames.ordinality())
{
ForEachItemIn(s, subfileLogicalFilenames)
dst.append(subfileLogicalFilenames.item(s));
}
if (mapping)
mapping->serializeMap(slave, dst);
else
CSlavePartMapping::serializeNullMap(dst);
}
virtual void validateFile(IDistributedFile *file)
{
IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *)queryHelper();
bool codeGenGrouped = 0 != (TDXgrouped & helper->getFlags());
bool isGrouped = fileDesc->isGrouped();
if (isGrouped != codeGenGrouped)
{
Owned<IException> e = MakeActivityWarning(&container, TE_GroupMismatch, "DFS and code generated group info. differs: DFS(%s), CodeGen(%s), using DFS info", isGrouped?"grouped":"ungrouped", codeGenGrouped?"grouped":"ungrouped");
queryJobChannel().fireException(e);
}
IOutputMetaData *recordSize = helper->queryDiskRecordSize()->querySerializedDiskMeta();
if (recordSize->isFixedSize()) // fixed size
{
if (0 != fileDesc->queryProperties().getPropInt("@recordSize"))
{
size32_t rSz = fileDesc->queryProperties().getPropInt("@recordSize");
if (isGrouped)
rSz--; // eog byte not to be included in this test.
if (rSz != recordSize->getMinRecordSize())
throw MakeThorException(TE_RecordSizeMismatch, "Published record size %d for file %s, does not match coded record size %d", rSz, fileName.get(), recordSize->getMinRecordSize());
}
if (!fileDesc->isCompressed() && (TDXcompress & helper->getFlags()))
{
size32_t rSz = recordSize->getMinRecordSize();
if (isGrouped) rSz++;
if (rSz >= MIN_ROWCOMPRESS_RECSIZE)
{
Owned<IException> e = MakeActivityWarning(&container, TE_CompressionMismatch, "Ignoring compression attribute on file '%s', which is not published as compressed in DFS", fileName.get());
queryJobChannel().fireException(e);
}
}
}
if (0 == (TDRnocrccheck & helper->getFlags()))
checkFormatCrc(this, file, helper->getFormatCrc(), false);
}
