void writeidxofs(offset_t o)
{
// lazy index write
if (outidx.get()) {
outidx->write(sizeof(o),&o);
return;
}
if (lastofs) {
if (fixedsize!=o-lastofs) {
// right create idx
StringBuffer tempname;
GetTempName(tempname.clear(),"srtidx",false);
outidxfile.setown(createIFile(tempname.str()));
Owned<IFileIO> fileioidx = outidxfile->open(IFOcreate);
outidx.setown(fileioidx?createBufferedIOStream(fileioidx,0x100000):NULL);
if (outidx.get()==NULL) {
StringBuffer err;
err.append("Cannot create ").append(outidxfile->queryFilename());
LOG(MCerror, thorJob, "%s", err.str());
throw MakeStringException(-1, "%s", err.str());
}
offset_t s = 0;
while (s<=lastofs) {
outidx->write(sizeof(s),&s);
s += fixedsize;
}
assertex(s==lastofs+fixedsize);
fixedsize = 0;
writeidxofs(o);
return;
}
}
else
fixedsize = (size32_t)(o-lastofs);
lastofs = o;
}
CDigitalSignatureManager(CLoadedKey *_pubKey, CLoadedKey *_privKey) : pubKey(_pubKey), privKey(_privKey)
{
signingConfigured = nullptr != privKey.get();
verifyingConfigured = nullptr != pubKey.get();
}
void prepareKey()
{
IDistributedFile *f = index.get();
IDistributedSuperFile *super = f->querySuperFile();
unsigned nparts = f->numParts(); // includes tlks if any, but unused in array
performPartLookup.ensure(nparts);
bool checkTLKConsistency = NULL != super && 0 != (TIRsorted & indexBaseHelper->getFlags());
if (nofilter)
{
while (nparts--) performPartLookup.append(true);
if (!checkTLKConsistency) return;
}
else
{
while (nparts--) performPartLookup.append(false); // parts to perform lookup set later
}
Owned<IDistributedFileIterator> iter;
if (super) {
iter.setown(super->getSubFileIterator(true));
verifyex(iter->first());
f = &iter->query();
}
unsigned width = f->numParts()-1;
assertex(width);
unsigned tlkCrc;
bool first = true;
unsigned superSubIndex=0;
bool fileCrc = false, rowCrc = false;
loop {
Owned<IDistributedFilePart> part = f->getPart(width);
if (checkTLKConsistency)
{
unsigned _tlkCrc;
if (part->getCrc(_tlkCrc))
fileCrc = true;
else if (part->queryAttributes().hasProp("@crc")) // NB: key "@crc" is not a crc on the file, but data within.
{
_tlkCrc = part->queryAttributes().getPropInt("@crc");
rowCrc = true;
}
else if (part->queryAttributes().hasProp("@tlkCrc")) // backward compat.
{
_tlkCrc = part->queryAttributes().getPropInt("@tlkCrc");
rowCrc = true;
}
else
{
if (rowCrc || fileCrc)
{
checkTLKConsistency = false;
Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, some crc attributes are missing", super->queryLogicalName());
container.queryJob().fireException(e);
}
}
if (rowCrc && fileCrc)
{
checkTLKConsistency = false;
Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, due to mixed crc types.", super->queryLogicalName());
container.queryJob().fireException(e);
}
if (checkTLKConsistency)
{
if (first)
{
tlkCrc = _tlkCrc;
first = false;
}
else if (tlkCrc != _tlkCrc)
throw MakeActivityException(this, 0, "Sorted output on super files comprising of non coparitioned sub keys is not supported (TLK's do not match)");
}
}
if (!nofilter)
{
Owned<IKeyIndex> keyIndex;
unsigned copy;
for (copy=0; copy<part->numCopies(); copy++)
{
RemoteFilename rfn;
OwnedIFile ifile = createIFile(part->getFilename(rfn,copy));
if (ifile->exists())
{
StringBuffer remotePath;
rfn.getRemotePath(remotePath);
unsigned crc;
part->getCrc(crc);
keyIndex.setown(createKeyIndex(remotePath.str(), crc, false, false));
break;
}
}
if (!keyIndex)
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", indexBaseHelper->getFileName());
unsigned maxSize = indexBaseHelper->queryDiskRecordSize()->querySerializedMeta()->getRecordSize(NULL); // used only if fixed
Owned <IKeyManager> tlk = createKeyManager(keyIndex, maxSize, NULL);
indexBaseHelper->createSegmentMonitors(tlk);
tlk->finishSegmentMonitors();
tlk->reset();
while (tlk->lookup(false))
{
if (tlk->queryFpos())
performPartLookup.replace(true, (aindex_t)(super?super->numSubFiles(true)*(tlk->queryFpos()-1)+superSubIndex:tlk->queryFpos()-1));
}
}
if (!super||!iter->next())
break;
superSubIndex++;
f = &iter->query();
if (width != f->numParts()-1)
throw MakeActivityException(this, 0, "Super key %s, with mixture of sub key width are not supported.", f->queryLogicalName());
}
}